| <!DOCTYPE HTML> |
| <html lang="en" class="light sidebar-visible" dir="ltr"> |
| <head> |
| <!-- Book generated using mdBook --> |
| <meta charset="UTF-8"> |
| <title>Rust Compiler Development Guide</title> |
| <meta name="robots" content="noindex"> |
| |
| |
| <!-- Custom HTML head --> |
| |
| <meta name="description" content="A guide to developing the Rust compiler (rustc)"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="theme-color" content="#ffffff"> |
| |
| <link rel="icon" href="favicon.svg"> |
| <link rel="shortcut icon" href="favicon.png"> |
| <link rel="stylesheet" href="css/variables.css"> |
| <link rel="stylesheet" href="css/general.css"> |
| <link rel="stylesheet" href="css/chrome.css"> |
| <link rel="stylesheet" href="css/print.css" media="print"> |
| |
| <!-- Fonts --> |
| <link rel="stylesheet" href="FontAwesome/css/font-awesome.css"> |
| <link rel="stylesheet" href="fonts/fonts.css"> |
| |
| <!-- Highlight.js Stylesheets --> |
| <link rel="stylesheet" id="highlight-css" href="highlight.css"> |
| <link rel="stylesheet" id="tomorrow-night-css" href="tomorrow-night.css"> |
| <link rel="stylesheet" id="ayu-highlight-css" href="ayu-highlight.css"> |
| |
| <!-- Custom theme stylesheets --> |
| <link rel="stylesheet" href="pagetoc.css"> |
| |
| |
| <!-- Provide site root and default themes to javascript --> |
| <script> |
| const path_to_root = ""; |
| const default_light_theme = "light"; |
| const default_dark_theme = "navy"; |
| window.path_to_searchindex_js = "searchindex.js"; |
| </script> |
| <!-- Start loading toc.js asap --> |
| <script src="toc.js"></script> |
| </head> |
| <body> |
| <div id="mdbook-help-container"> |
| <div id="mdbook-help-popup"> |
| <h2 class="mdbook-help-title">Keyboard shortcuts</h2> |
| <div> |
| <p>Press <kbd>←</kbd> or <kbd>→</kbd> to navigate between chapters</p> |
| <p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p> |
| <p>Press <kbd>?</kbd> to show this help</p> |
| <p>Press <kbd>Esc</kbd> to hide this help</p> |
| </div> |
| </div> |
| </div> |
| <div id="body-container"> |
| <!-- Work around some values being stored in localStorage wrapped in quotes --> |
| <script> |
| try { |
| let theme = localStorage.getItem('mdbook-theme'); |
| let sidebar = localStorage.getItem('mdbook-sidebar'); |
| |
| if (theme.startsWith('"') && theme.endsWith('"')) { |
| localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); |
| } |
| |
| if (sidebar.startsWith('"') && sidebar.endsWith('"')) { |
| localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); |
| } |
| } catch (e) { } |
| </script> |
| |
| <!-- Set the theme before any content is loaded, prevents flash --> |
| <script> |
| const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme; |
| let theme; |
| try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } |
| if (theme === null || theme === undefined) { theme = default_theme; } |
| const html = document.documentElement; |
| html.classList.remove('light') |
| html.classList.add(theme); |
| html.classList.add("js"); |
| </script> |
| |
| <input type="checkbox" id="sidebar-toggle-anchor" class="hidden"> |
| |
| <!-- Hide / unhide sidebar before it is displayed --> |
| <script> |
| let sidebar = null; |
| const sidebar_toggle = document.getElementById("sidebar-toggle-anchor"); |
| if (document.body.clientWidth >= 1080) { |
| try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } |
| sidebar = sidebar || 'visible'; |
| } else { |
| sidebar = 'hidden'; |
| sidebar_toggle.checked = false; |
| } |
| if (sidebar === 'visible') { |
| sidebar_toggle.checked = true; |
| } else { |
| html.classList.remove('sidebar-visible'); |
| } |
| </script> |
| |
| <nav id="sidebar" class="sidebar" aria-label="Table of contents"> |
| <!-- populated by js --> |
| <mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox> |
| <noscript> |
| <iframe class="sidebar-iframe-outer" src="toc.html"></iframe> |
| </noscript> |
| <div id="sidebar-resize-handle" class="sidebar-resize-handle"> |
| <div class="sidebar-resize-indicator"></div> |
| </div> |
| </nav> |
| |
| <div id="page-wrapper" class="page-wrapper"> |
| |
| <div class="page"> |
| <div id="menu-bar-hover-placeholder"></div> |
| <div id="menu-bar" class="menu-bar sticky"> |
| <div class="left-buttons"> |
| <label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> |
| <i class="fa fa-bars"></i> |
| </label> |
| <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> |
| <i class="fa fa-paint-brush"></i> |
| </button> |
| <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> |
| <li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="light">Light</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> |
| </ul> |
| <button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar"> |
| <i class="fa fa-search"></i> |
| </button> |
| </div> |
| |
| <h1 class="menu-title">Rust Compiler Development Guide</h1> |
| |
| <div class="right-buttons"> |
| <a href="print.html" title="Print this book" aria-label="Print this book"> |
| <i id="print-button" class="fa fa-print"></i> |
| </a> |
| <a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository"> |
| <i id="git-repository-button" class="fa fa-github"></i> |
| </a> |
| |
| </div> |
| </div> |
| |
| <div id="search-wrapper" class="hidden"> |
| <form id="searchbar-outer" class="searchbar-outer"> |
| <div class="search-wrapper"> |
| <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> |
| <div class="spinner-wrapper"> |
| <i class="fa fa-spinner fa-spin"></i> |
| </div> |
| </div> |
| </form> |
| <div id="searchresults-outer" class="searchresults-outer hidden"> |
| <div id="searchresults-header" class="searchresults-header"></div> |
| <ul id="searchresults"> |
| </ul> |
| </div> |
| </div> |
| |
| <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> |
| <script> |
| document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); |
| document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); |
| Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { |
| link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); |
| }); |
| </script> |
| |
| <div id="content" class="content"> |
| <main> |
| <h1 id="getting-started"><a class="header" href="#getting-started">Getting Started</a></h1> |
| <p>Thank you for your interest in contributing to Rust! |
| There are many ways to contribute, and we appreciate all of them.</p> |
| <p>If this is your first time contributing, the <a href="./walkthrough.html">walkthrough</a> chapter can give you a good example of |
| how a typical contribution would go.</p> |
| <p>This documentation is <em>not</em> intended to be comprehensive; |
| it is meant to be a quick guide for the most useful things. |
| For more information, |
| see <a href="building/how-to-build-and-run.html">How to build and run the compiler</a>.</p> |
| <h2 id="asking-questions"><a class="header" href="#asking-questions">Asking Questions</a></h2> |
| <p>If you have questions, please make a post on the <a href="https://rust-lang.zulipchat.com">Rust Zulip server</a> or |
| <a href="https://internals.rust-lang.org">internals.rust-lang.org</a>. |
| See the <a href="https://www.rust-lang.org/governance">list of teams and working groups</a> and <a href="https://www.rust-lang.org/community">the Community page</a> on the |
| official website for more resources.</p> |
| <p>As a reminder, all contributors are expected to follow our <a href="https://www.rust-lang.org/policies/code-of-conduct">Code of Conduct</a>.</p> |
| <p>The compiler team (or <code>t-compiler</code>) usually hangs out in Zulip in |
| <a href="https://rust-lang.zulipchat.com/#narrow/channel/131828-t-compiler">the #t-compiler channel</a>; |
| questions about how the compiler works can go in <a href="https://rust-lang.zulipchat.com/#narrow/channel/182449-t-compiler.2Fhelp">#t-compiler/help</a>.</p> |
| <p><strong>Please ask questions!</strong> A lot of people report feeling that they are "wasting |
| expert's time", but nobody on <code>t-compiler</code> feels this way. |
| Contributors are important to us.</p> |
| <p>Also, if you feel comfortable, prefer public topics, as this means others can |
| see the questions and answers, and perhaps even integrate them back into this guide :)</p> |
| <p><strong>Tip</strong>: If you're not a native English speaker and feel unsure about writing, try using a translator to help. |
| But avoid using LLM tools that generate long, complex words. |
| In daily teamwork, <strong>simple and clear words</strong> are best for easy understanding. |
| Even small typos or grammar mistakes can make you seem more human, and people connect better with humans.</p> |
| <h3 id="experts"><a class="header" href="#experts">Experts</a></h3> |
| <p>Not all <code>t-compiler</code> members are experts on all parts of <code>rustc</code>; |
| it's a pretty large project. |
| To find out who could have some expertise on |
| different parts of the compiler, <a href="https://github.com/rust-lang/rust/blob/HEAD/triagebot.toml">consult triagebot assign groups</a>. |
| The sections that start with <code>[assign*</code> in <code>triagebot.toml</code> file. |
| But also, feel free to ask questions even if you can't figure out who to ping.</p> |
| <p>Another way to find experts for a given part of the compiler is to see who has made recent commits. |
| For example, to find people who have recently worked on name resolution since the 1.68.2 release, |
| you could run <code>git shortlog -n 1.68.2.. compiler/rustc_resolve/</code>. |
| Ignore any commits starting with |
| "Rollup merge" or commits by <code>@bors</code> (see <a href="./contributing.html#ci">CI contribution procedures</a> for |
| more information about these commits).</p> |
| <h3 id="etiquette"><a class="header" href="#etiquette">Etiquette</a></h3> |
| <p>We do ask that you be mindful to include as much useful information as you can |
| in your question, but we recognize this can be hard if you are unfamiliar with contributing to Rust.</p> |
| <p>Just pinging someone without providing any context can be a bit annoying and |
| just create noise, so we ask that you be mindful of the fact that the |
| <code>t-compiler</code> folks get a lot of pings in a day.</p> |
| <h2 id="what-should-i-work-on"><a class="header" href="#what-should-i-work-on">What should I work on?</a></h2> |
| <p>The Rust project is quite large and it can be difficult to know which parts of the project need |
| help, or are a good starting place for beginners. |
| Here are some suggested starting places.</p> |
| <h3 id="easy-or-mentored-issues"><a class="header" href="#easy-or-mentored-issues">Easy or mentored issues</a></h3> |
| <p>If you're looking for somewhere to start, check out the following <a href="https://github.com/issues?q=is%3Aopen+is%3Aissue+org%3Arust-lang+no%3Aassignee+label%3AE-easy%2C%22good+first+issue%22%2Cgood-first-issue%2CE-medium%2CEasy%2CE-help-wanted%2CE-mentor+-label%3AS-blocked+-linked%3Apr+">issue |
| search</a>. |
| See the <a href="./contributing.html#issue-triage">Triage</a> for an explanation of these labels. |
| You can also try filtering the search to areas you're interested in. |
| For example:</p> |
| <ul> |
| <li><code>repo:rust-lang/rust-clippy</code> will only show clippy issues</li> |
| <li><code>label:T-compiler</code> will only show issues related to the compiler</li> |
| <li><code>label:A-diagnostics</code> will only show diagnostic issues</li> |
| </ul> |
| <p>Not all important or beginner work has issue labels. |
| See below for how to find work that isn't labelled.</p> |
| <h3 id="recurring-work"><a class="header" href="#recurring-work">Recurring work</a></h3> |
| <p>Some work is too large to be done by a single person. |
| In this case, it's common to have "Tracking issues" to co-ordinate the work between contributors. |
| Here are some example tracking issues where |
| it's easy to pick up work without a large time commitment:</p> |
| <ul> |
| <li><em>Add recurring work items here.</em></li> |
| </ul> |
| <p>If you find more recurring work, please feel free to add it here!</p> |
| <h3 id="clippy-issues"><a class="header" href="#clippy-issues">Clippy issues</a></h3> |
| <p>The <a href="https://doc.rust-lang.org/clippy/">Clippy</a> project has spent a long time making its contribution process as friendly to newcomers |
| as possible. |
| Consider working on it first to get familiar with the process and the compiler internals.</p> |
| <p>See <a href="https://github.com/rust-lang/rust-clippy/blob/master/CONTRIBUTING.md">the Clippy contribution guide</a> for instructions on getting started.</p> |
| <h3 id="diagnostic-issues"><a class="header" href="#diagnostic-issues">Diagnostic issues</a></h3> |
| <p>Many diagnostic issues are self-contained and don't need detailed background knowledge of the |
| compiler. |
| You can see a list of diagnostic issues <a href="https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AA-diagnostics+no%3Aassignee">here</a>.</p> |
| <h3 id="picking-up-abandoned-pull-requests"><a class="header" href="#picking-up-abandoned-pull-requests">Picking up abandoned pull requests</a></h3> |
| <p>Sometimes, contributors send a pull request, but later find out that they don't have enough |
| time to work on it, or they simply are not interested in it anymore. |
| Such PRs are often eventually closed and they receive the <code>S-inactive</code> label. |
| You could try to examine some of these PRs and pick up the work. |
| You can find the list of such PRs <a href="https://github.com/rust-lang/rust/pulls?q=is%3Apr+label%3AS-inactive+is%3Aclosed">here</a>.</p> |
| <p>If the PR has been implemented in some other way in the meantime, the <code>S-inactive</code> label |
| should be removed from it. |
| If not, and it seems that there is still interest in the change, |
| you can try to rebase the pull request on top of the latest <code>main</code> branch and send a new |
| pull request, continuing the work on the feature.</p> |
| <h3 id="writing-tests"><a class="header" href="#writing-tests">Writing tests</a></h3> |
| <p>Issues that have been resolved but do not have a regression test are marked with the <code>E-needs-test</code> label. |
| Writing unit tests is a low-risk, |
| lower-priority task that offers new contributors a great opportunity to familiarize themselves |
| with the testing infrastructure and contribution workflow.</p> |
| <h3 id="contributing-to-std-standard-library"><a class="header" href="#contributing-to-std-standard-library">Contributing to std (standard library)</a></h3> |
| <p>See <a href="https://std-dev-guide.rust-lang.org/">std-dev-guide</a>.</p> |
| <h3 id="contributing-code-to-other-rust-projects"><a class="header" href="#contributing-code-to-other-rust-projects">Contributing code to other Rust projects</a></h3> |
| <p>There are a bunch of other projects that you can contribute to outside of the |
| <code>rust-lang/rust</code> repo, including <code>cargo</code>, <code>miri</code>, <code>rustup</code>, and many others.</p> |
| <p>These repos might have their own contributing guidelines and procedures. |
| Many of them are owned by working groups. |
| For more info, see the documentation in those repos' READMEs.</p> |
| <h3 id="other-ways-to-contribute"><a class="header" href="#other-ways-to-contribute">Other ways to contribute</a></h3> |
| <p>There are a bunch of other ways you can contribute, especially if you don't |
| feel comfortable jumping straight into the large <code>rust-lang/rust</code> codebase.</p> |
| <p>The following tasks are doable without much background knowledge but are incredibly helpful:</p> |
| <ul> |
| <li><a href="./contributing.html#writing-documentation">Writing documentation</a>: if you are feeling a bit more intrepid, you could try |
| to read a part of the code and write doc comments for it. |
| This will help you to learn some part of the compiler while also producing a useful artifact!</li> |
| <li><a href="./contributing.html#issue-triage">Triaging issues</a>: categorizing, replicating, and minimizing issues is very helpful to the Rust maintainers.</li> |
| <li><a href="https://rust-lang.github.io/compiler-team/working-groups/">Working groups</a>: there are a bunch of working groups on a wide variety |
| of rust-related things.</li> |
| <li>Answer questions on <a href="https://users.rust-lang.org/">users.rust-lang.org</a>, or on <a href="http://stackoverflow.com/questions/tagged/rust">Stack Overflow</a>.</li> |
| <li>Participate in the <a href="https://github.com/rust-lang/rfcs">RFC process</a>.</li> |
| <li>Find a <a href="https://github.com/rust-lang/rfcs/labels/A-community-library">requested community library</a>, build it, and publish |
| it to <a href="http://crates.io">Crates.io</a>. |
| Easier said than done, but very, very valuable!</li> |
| </ul> |
| <h2 id="cloning-and-building"><a class="header" href="#cloning-and-building">Cloning and Building</a></h2> |
| <p>See <a href="./building/how-to-build-and-run.html">"How to build and run the compiler"</a>.</p> |
| <h2 id="contributor-procedures"><a class="header" href="#contributor-procedures">Contributor Procedures</a></h2> |
| <p>This section has moved to the <a href="./contributing.html">"Contribution Procedures"</a> chapter.</p> |
| <h2 id="other-resources"><a class="header" href="#other-resources">Other Resources</a></h2> |
| <p>This section has moved to the <a href="./about-this-guide.html#other-places-to-find-information">"About this guide"</a> chapter.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="about-this-guide"><a class="header" href="#about-this-guide">About this guide</a></h1> |
| <p>This guide is meant to help document how rustc – the Rust compiler – works, |
| as well as to help new contributors get involved in rustc development.</p> |
| <p>There are several parts to this guide:</p> |
| <ol> |
| <li><a href="./building/how-to-build-and-run.html">Building and debugging <code>rustc</code></a>: |
| Contains information that should be useful no matter how you are contributing, |
| about building, debugging, profiling, etc.</li> |
| <li><a href="./contributing.html">Contributing to Rust</a>: |
| Contains information that should be useful no matter how you are contributing, |
| about procedures for contribution, using git and Github, stabilizing features, etc.</li> |
| <li><a href="./building/bootstrapping/intro.html">Bootstrapping</a>: |
| Describes how the Rust compiler builds itself using previous versions, including |
| an introduction to the bootstrap process and debugging methods.</li> |
| <li><a href="./part-2-intro.html">High-level Compiler Architecture</a>: |
| Discusses the high-level architecture of the compiler and stages of the compile process.</li> |
| <li><a href="./part-3-intro.html">Source Code Representation</a>: |
| Describes the process of taking raw source code from the user |
| and transforming it into various forms that the compiler can work with easily.</li> |
| <li><a href="./cli.html">Supporting Infrastructure</a>: |
| Covers command-line argument conventions, compiler entry points like rustc_driver and |
| rustc_interface, and the design and implementation of errors and lints.</li> |
| <li><a href="./part-4-intro.html">Analysis</a>: |
| Discusses the analyses that the compiler uses to check various properties of the code |
| and inform later stages of the compile process (e.g., type checking).</li> |
| <li><a href="./part-5-intro.html">MIR to Binaries</a>: How linked executable machine code is generated.</li> |
| <li><a href="./appendix/background.html">Appendices</a> at the end with useful reference information. |
| There are a few of these with different information, including a glossary.</li> |
| </ol> |
| <h3 id="constant-change"><a class="header" href="#constant-change">Constant change</a></h3> |
| <p>Keep in mind that <code>rustc</code> is a real production-quality product, |
| being worked upon continuously by a sizeable set of contributors. |
| As such, it has its fair share of codebase churn and technical debt. |
| In addition, many of the ideas discussed throughout this guide are idealized designs |
| that are not fully realized yet. |
| All this makes keeping this guide completely up to date on everything very hard!</p> |
| <p>The guide itself is of course open source as well, |
| and the sources are hosted on <a href="https://github.com/rust-lang/rustc-dev-guide/">a GitHub repository</a>. |
| If you find any mistakes in the guide, please file an issue. |
| Even better, open a PR with a correction!</p> |
| <p>If you do contribute to the guide, |
| please see the corresponding <a href="contributing.html#contributing-to-rustc-dev-guide">subsection on writing documentation in this guide</a>.</p> |
| <blockquote> |
| <p>“‘All conditioned things are impermanent’ — |
| when one sees this with wisdom, one turns away from suffering.” |
| <em>The Dhammapada, verse 277</em></p> |
| </blockquote> |
| <h2 id="other-places-to-find-information"><a class="header" href="#other-places-to-find-information">Other places to find information</a></h2> |
| <p>You might also find the following sites useful:</p> |
| <ul> |
| <li>This guide contains information about how various parts of the |
| compiler work and how to contribute to the compiler.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle">rustc API docs</a> -- rustdoc documentation for the compiler, devtools, and internal tools</li> |
| <li><a href="https://forge.rust-lang.org/">Forge</a> -- contains documentation about Rust infrastructure, team procedures, and more</li> |
| <li><a href="https://github.com/rust-lang/compiler-team/">compiler-team</a> -- the home-base for the Rust compiler team, with description |
| of the team procedures, active working groups, and the team calendar.</li> |
| <li><a href="https://std-dev-guide.rust-lang.org/">std-dev-guide</a> -- a similar guide for developing the standard library.</li> |
| <li><a href="https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler">The t-compiler Zulip</a></li> |
| <li>The <a href="http://internals.rust-lang.org">Rust Internals forum</a>, a place to ask questions and |
| discuss Rust's internals</li> |
| <li>The <a href="https://doc.rust-lang.org/book/">Rust reference</a>, even though it doesn't specifically talk about |
| Rust's internals, is a great resource nonetheless</li> |
| <li>Although out of date, <a href="https://tomlee.co/2014/04/a-more-detailed-tour-of-the-rust-compiler/">Tom Lee's great blog article</a> is very helpful</li> |
| <li>The <a href="tests/intro.html">Rust Compiler Testing Docs</a></li> |
| <li>For <a href="https://github.com/bors">@bors</a>, <a href="https://bors.rust-lang.org/">this cheat sheet</a> is helpful</li> |
| <li>Google is always helpful when programming. |
| You can <a href="https://www.google.com/search?q=site:doc.rust-lang.org+your+query+here">search all Rust documentation</a> (the standard library, |
| the compiler, the books, the references, and the guides) to quickly find |
| information about the language and compiler.</li> |
| <li>You can also use Rustdoc's built-in search feature to find documentation on |
| types and functions within the crates you're looking at. You can also search |
| by type signature! For example, searching for <code>* -> vec</code> should find all |
| functions that return a <code>Vec<T></code>. |
| <em>Hint:</em> Find more tips and keyboard shortcuts by typing <code>?</code> on any Rustdoc |
| page!</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="how-to-build-and-run-the-compiler"><a class="header" href="#how-to-build-and-run-the-compiler">How to build and run the compiler</a></h1> |
| <div class="warning"> |
| <p>For <code>profile = "library"</code> users, or users who use <code>download-rustc = true | "if-unchanged"</code>, please be advised that |
| the <code>./x test library/std</code> flow where <code>download-rustc</code> is active (i.e. no compiler changes) is currently broken. |
| This is tracked in <a href="https://github.com/rust-lang/rust/issues/142505">https://github.com/rust-lang/rust/issues/142505</a>. Only the <code>./x test</code> flow is affected in this |
| case, <code>./x {check,build} library/std</code> should still work.</p> |
| <p>In the short-term, you may need to disable <code>download-rustc</code> for <code>./x test library/std</code>. This can be done either by:</p> |
| <ol> |
| <li><code>./x test library/std --set rust.download-rustc=false</code></li> |
| <li>Or set <code>rust.download-rustc=false</code> in <code>bootstrap.toml</code>.</li> |
| </ol> |
| <p>Unfortunately that will require building the stage 1 compiler. The bootstrap team is working on this, but |
| implementing a maintainable fix is taking some time.</p> |
| </div> |
| <p>The compiler is built using a tool called <code>x.py</code>. You will need to |
| have Python installed to run it.</p> |
| <h2 id="quick-start"><a class="header" href="#quick-start">Quick Start</a></h2> |
| <p>For a less in-depth quick-start of getting the compiler running, see <a href="building/./quickstart.html">quickstart</a>.</p> |
| <h2 id="get-the-source-code"><a class="header" href="#get-the-source-code">Get the source code</a></h2> |
| <p>The main repository is <a href="https://github.com/rust-lang/rust"><code>rust-lang/rust</code></a>. This contains the compiler, |
| the standard library (including <code>core</code>, <code>alloc</code>, <code>test</code>, <code>proc_macro</code>, etc), |
| and a bunch of tools (e.g. <code>rustdoc</code>, the bootstrapping infrastructure, etc).</p> |
| <p>The very first step to work on <code>rustc</code> is to clone the repository:</p> |
| <pre><code class="language-bash">git clone https://github.com/rust-lang/rust.git |
| cd rust |
| </code></pre> |
| <h3 id="partial-clone-the-repository"><a class="header" href="#partial-clone-the-repository">Partial clone the repository</a></h3> |
| <p>Due to the size of the repository, cloning on a slower internet connection can take a long time, |
| and requires disk space to store the full history of every file and directory. |
| Instead, it is possible to tell git to perform a <em>partial clone</em>, which will only fully retrieve |
| the current file contents, but will automatically retrieve further file contents when you, e.g., |
| jump back in the history. |
| All git commands will continue to work as usual, at the price of requiring an internet connection |
| to visit not-yet-loaded points in history.</p> |
| <pre><code class="language-bash">git clone --filter='blob:none' https://github.com/rust-lang/rust.git |
| cd rust |
| </code></pre> |
| <blockquote> |
| <p><strong>NOTE</strong>: <a href="https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/">This link</a> |
| describes this type of checkout in more detail, and also compares it to other modes, such as |
| shallow cloning.</p> |
| </blockquote> |
| <h3 id="shallow-clone-the-repository"><a class="header" href="#shallow-clone-the-repository">Shallow clone the repository</a></h3> |
| <p>An older alternative to partial clones is to use shallow clone the repository instead. |
| To do so, you can use the <code>--depth N</code> option with the <code>git clone</code> command. |
| This instructs <code>git</code> to perform a "shallow clone", cloning the repository but truncating it to |
| the last <code>N</code> commits.</p> |
| <p>Passing <code>--depth 1</code> tells <code>git</code> to clone the repository but truncate the history to the latest |
| commit that is on the <code>main</code> branch, which is usually fine for browsing the source code or |
| building the compiler.</p> |
| <pre><code class="language-bash">git clone --depth 1 https://github.com/rust-lang/rust.git |
| cd rust |
| </code></pre> |
| <blockquote> |
| <p><strong>NOTE</strong>: A shallow clone limits which <code>git</code> commands can be run. |
| If you intend to work on and contribute to the compiler, it is |
| generally recommended to fully clone the repository <a href="building/how-to-build-and-run.html#get-the-source-code">as shown above</a>, |
| or to perform a <a href="building/how-to-build-and-run.html#partial-clone-the-repository">partial clone</a> instead.</p> |
| <p>For example, <code>git bisect</code> and <code>git blame</code> require access to the commit history, |
| so they don't work if the repository was cloned with <code>--depth 1</code>.</p> |
| </blockquote> |
| <h2 id="what-is-xpy"><a class="header" href="#what-is-xpy">What is <code>x.py</code>?</a></h2> |
| <p><code>x.py</code> is the build tool for the <code>rust</code> repository. It can build docs, run tests, and compile the |
| compiler and standard library.</p> |
| <p>This chapter focuses on the basics to be productive, but |
| if you want to learn more about <code>x.py</code>, <a href="building/./bootstrapping/intro.html">read this chapter</a>.</p> |
| <p>Also, using <code>x</code> rather than <code>x.py</code> is recommended as:</p> |
| <blockquote> |
| <p><code>./x</code> is the most likely to work on every system (on Unix it runs the shell script |
| that does python version detection, on Windows it will probably run the |
| powershell script - certainly less likely to break than <code>./x.py</code> which often just |
| opens the file in an editor).<sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup></p> |
| </blockquote> |
| <p>(You can find the platform related scripts around the <code>x.py</code>, like <code>x.ps1</code>)</p> |
| <p>Notice that this is not absolute. For instance, using Nushell in VSCode on Win10, |
| typing <code>x</code> or <code>./x</code> still opens <code>x.py</code> in an editor rather than invoking the program. :)</p> |
| <p>In the rest of this guide, we use <code>x</code> rather than <code>x.py</code> directly. The following |
| command:</p> |
| <pre><code class="language-bash">./x check |
| </code></pre> |
| <p>could be replaced by:</p> |
| <pre><code class="language-bash">./x.py check |
| </code></pre> |
| <h3 id="running-xpy"><a class="header" href="#running-xpy">Running <code>x.py</code></a></h3> |
| <p>The <code>x.py</code> command can be run directly on most Unix systems in the following format:</p> |
| <pre><code class="language-sh">./x <subcommand> [flags] |
| </code></pre> |
| <p>This is how the documentation and examples assume you are running <code>x.py</code>. |
| Some alternative ways are:</p> |
| <pre><code class="language-sh"># On a Unix shell if you don't have the necessary `python3` command |
| ./x <subcommand> [flags] |
| |
| # In Windows Powershell (if powershell is configured to run scripts) |
| ./x <subcommand> [flags] |
| ./x.ps1 <subcommand> [flags] |
| |
| # On the Windows Command Prompt (if .py files are configured to run Python) |
| x.py <subcommand> [flags] |
| |
| # You can also run Python yourself, e.g.: |
| python x.py <subcommand> [flags] |
| </code></pre> |
| <p>On Windows, the Powershell commands may give you an error that looks like this:</p> |
| <pre><code>PS C:\Users\vboxuser\rust> ./x |
| ./x : File C:\Users\vboxuser\rust\x.ps1 cannot be loaded because running scripts is disabled on this system. For more |
| information, see about_Execution_Policies at https://go.microsoft.com/fwlink/?LinkID=135170. |
| At line:1 char:1 |
| + ./x |
| + ~~~ |
| + CategoryInfo : SecurityError: (:) [], PSSecurityException |
| + FullyQualifiedErrorId : UnauthorizedAccess |
| </code></pre> |
| <p>You can avoid this error by allowing powershell to run local scripts:</p> |
| <pre><code>Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser |
| </code></pre> |
| <h4 id="running-xpy-slightly-more-conveniently"><a class="header" href="#running-xpy-slightly-more-conveniently">Running <code>x.py</code> slightly more conveniently</a></h4> |
| <p>There is a binary that wraps <code>x.py</code> called <code>x</code> in <code>src/tools/x</code>. All it does is |
| run <code>x.py</code>, but it can be installed system-wide and run from any subdirectory |
| of a checkout. It also looks up the appropriate version of <code>python</code> to use.</p> |
| <p>You can install it with <code>cargo install --path src/tools/x</code>.</p> |
| <p>To clarify that this is another global installed binary util, which is |
| similar to the one declared in section <a href="building/how-to-build-and-run.html#what-is-xpy">What is <code>x.py</code></a>, but |
| it works as an independent process to execute the <code>x.py</code> rather than calling the |
| shell to run the platform related scripts.</p> |
| <h2 id="create-a-bootstraptoml"><a class="header" href="#create-a-bootstraptoml">Create a <code>bootstrap.toml</code></a></h2> |
| <p>To start, run <code>./x setup</code> and select the <code>compiler</code> defaults. This will do some initialization |
| and create a <code>bootstrap.toml</code> for you with reasonable defaults. If you use a different default (which |
| you'll likely want to do if you want to contribute to an area of rust other than the compiler, such |
| as rustdoc), make sure to read information about that default (located in <code>src/bootstrap/defaults</code>) |
| as the build process may be different for other defaults.</p> |
| <p>Alternatively, you can write <code>bootstrap.toml</code> by hand. See <code>bootstrap.example.toml</code> for all the available |
| settings and explanations of them. See <code>src/bootstrap/defaults</code> for common settings to change.</p> |
| <p>If you have already built <code>rustc</code> and you change settings related to LLVM, then you may have to |
| execute <code>./x clean --all</code> for subsequent configuration changes to take effect. Note that <code>./x clean</code> will not cause a rebuild of LLVM.</p> |
| <h2 id="common-x-commands"><a class="header" href="#common-x-commands">Common <code>x</code> commands</a></h2> |
| <p>Here are the basic invocations of the <code>x</code> commands most commonly used when |
| working on <code>rustc</code>, <code>std</code>, <code>rustdoc</code>, and other tools.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Command</th><th>When to use it</th></tr></thead><tbody> |
| <tr><td><code>./x check</code></td><td>Quick check to see if most things compile; <a href="building/suggested.html#configuring-rust-analyzer-for-rustc">rust-analyzer can run this automatically for you</a></td></tr> |
| <tr><td><code>./x build</code></td><td>Builds <code>rustc</code>, <code>std</code>, and <code>rustdoc</code></td></tr> |
| <tr><td><code>./x test</code></td><td>Runs all tests</td></tr> |
| <tr><td><code>./x fmt</code></td><td>Formats all code</td></tr> |
| </tbody></table> |
| </div> |
| <p>As written, these commands are reasonable starting points. However, there are |
| additional options and arguments for each of them that are worth learning for |
| serious development work. In particular, <code>./x build</code> and <code>./x test</code> |
| provide many ways to compile or test a subset of the code, which can save a lot |
| of time.</p> |
| <p>Also, note that <code>x</code> supports all kinds of path suffixes for <code>compiler</code>, <code>library</code>, |
| and <code>src/tools</code> directories. So, you can simply run <code>x test tidy</code> instead of |
| <code>x test src/tools/tidy</code>. Or, <code>x build std</code> instead of <code>x build library/std</code>.</p> |
| <p>See the chapters on |
| <a href="building/../tests/running.html">testing</a> and <a href="building/../rustdoc.html">rustdoc</a> for more details.</p> |
| <h3 id="building-the-compiler"><a class="header" href="#building-the-compiler">Building the compiler</a></h3> |
| <p>Note that building will require a relatively large amount of storage space. |
| You may want to have upwards of 10 or 15 gigabytes available to build the compiler.</p> |
| <p>Once you've created a <code>bootstrap.toml</code>, you are now ready to run |
| <code>x</code>. There are a lot of options here, but let's start with what is |
| probably the best "go to" command for building a local compiler:</p> |
| <pre><code class="language-console">./x build library |
| </code></pre> |
| <p>What this command does is:</p> |
| <ul> |
| <li>Build <code>rustc</code> using the stage0 compiler and stage0 <code>std</code>.</li> |
| <li>Build <code>library</code> (the standard libraries) with the stage1 compiler that was just built.</li> |
| <li>Assemble a working stage1 sysroot, containing the stage1 compiler and stage1 standard libraries.</li> |
| </ul> |
| <p>This final product (stage1 compiler + libs built using that compiler) |
| is what you need to build other Rust programs (unless you use <code>#![no_std]</code> or |
| <code>#![no_core]</code>).</p> |
| <p>You will probably find that building the stage1 <code>std</code> is a bottleneck for you, |
| but fear not, there is a (hacky) workaround... |
| see <a href="building/./suggested.html#faster-rebuilds-with---keep-stage-std">the section on avoiding rebuilds for std</a>.</p> |
| <p>Sometimes you don't need a full build. When doing some kind of |
| "type-based refactoring", like renaming a method, or changing the |
| signature of some function, you can use <code>./x check</code> instead for a much faster build.</p> |
| <p>Note that this whole command just gives you a subset of the full <code>rustc</code> |
| build. The <strong>full</strong> <code>rustc</code> build (what you get with <code>./x build --stage 2 rustc</code>) has quite a few more steps:</p> |
| <ul> |
| <li>Build <code>rustc</code> with the stage1 compiler. |
| <ul> |
| <li>The resulting compiler here is called the "stage2" compiler, which uses stage1 std from the previous command.</li> |
| </ul> |
| </li> |
| <li>Build <code>librustdoc</code> and a bunch of other things with the stage2 compiler.</li> |
| </ul> |
| <p>You almost never need to do this.</p> |
| <h3 id="build-specific-components"><a class="header" href="#build-specific-components">Build specific components</a></h3> |
| <p>If you are working on the standard library, you probably don't need to build |
| every other default component. Instead, you can build a specific component by |
| providing its name, like this:</p> |
| <pre><code class="language-bash">./x build --stage 1 library |
| </code></pre> |
| <p>If you choose the <code>library</code> profile when running <code>x setup</code>, you can omit <code>--stage 1</code> (it's the |
| default).</p> |
| <h2 id="creating-a-rustup-toolchain"><a class="header" href="#creating-a-rustup-toolchain">Creating a rustup toolchain</a></h2> |
| <p>Once you have successfully built <code>rustc</code>, you will have created a bunch |
| of files in your <code>build</code> directory. In order to actually run the |
| resulting <code>rustc</code>, we recommend creating rustup toolchains. The first |
| one will run the stage1 compiler (which we built above). The second |
| will execute the stage2 compiler (which we did not build, but which |
| you will likely need to build at some point; for example, if you want |
| to run the entire test suite).</p> |
| <pre><code class="language-bash">rustup toolchain link stage1 build/host/stage1 |
| rustup toolchain link stage2 build/host/stage2 |
| </code></pre> |
| <p>Now you can run the <code>rustc</code> you built with. If you run with <code>-vV</code>, you |
| should see a version number ending in <code>-dev</code>, indicating a build from |
| your local environment:</p> |
| <pre><code class="language-bash">$ rustc +stage1 -vV |
| rustc 1.48.0-dev |
| binary: rustc |
| commit-hash: unknown |
| commit-date: unknown |
| host: x86_64-unknown-linux-gnu |
| release: 1.48.0-dev |
| LLVM version: 11.0 |
| </code></pre> |
| <p>The rustup toolchain points to the specified toolchain compiled in your <code>build</code> directory, |
| so the rustup toolchain will be updated whenever <code>x build</code> or <code>x test</code> are run for |
| that toolchain/stage.</p> |
| <p><strong>Note:</strong> the toolchain we've built does not include <code>cargo</code>. In this case, <code>rustup</code> will |
| fall back to using <code>cargo</code> from the installed <code>nightly</code>, <code>beta</code>, or <code>stable</code> toolchain |
| (in that order). If you need to use unstable <code>cargo</code> flags, be sure to run |
| <code>rustup install nightly</code> if you haven't already. See the |
| <a href="https://rust-lang.github.io/rustup/concepts/toolchains.html#custom-toolchains">rustup documentation on custom toolchains</a>.</p> |
| <p><strong>Note:</strong> rust-analyzer and IntelliJ Rust plugin use a component called |
| <code>rust-analyzer-proc-macro-srv</code> to work with proc macros. If you intend to use a |
| custom toolchain for a project (e.g. via <code>rustup override set stage1</code>) you may |
| want to build this component:</p> |
| <pre><code class="language-bash">./x build proc-macro-srv-cli |
| </code></pre> |
| <h2 id="building-targets-for-cross-compilation"><a class="header" href="#building-targets-for-cross-compilation">Building targets for cross-compilation</a></h2> |
| <p>To produce a compiler that can cross-compile for other targets, |
| pass any number of <code>target</code> flags to <code>x build</code>. |
| For example, if your host platform is <code>x86_64-unknown-linux-gnu</code> |
| and your cross-compilation target is <code>wasm32-wasip1</code>, you can build with:</p> |
| <pre><code class="language-bash">./x build --target x86_64-unknown-linux-gnu,wasm32-wasip1 |
| </code></pre> |
| <p>Note that if you want the resulting compiler to be able to build crates that |
| involve proc macros or build scripts, you must be sure to explicitly build target support for the |
| host platform (in this case, <code>x86_64-unknown-linux-gnu</code>).</p> |
| <p>If you want to always build for other targets without needing to pass flags to <code>x build</code>, |
| you can configure this in the <code>[build]</code> section of your <code>bootstrap.toml</code> like so:</p> |
| <pre><code class="language-toml">[build] |
| target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1"] |
| </code></pre> |
| <p>Note that building for some targets requires having external dependencies installed |
| (e.g. building musl targets requires a local copy of musl). |
| Any target-specific configuration (e.g. the path to a local copy of musl) |
| will need to be provided by your <code>bootstrap.toml</code>. |
| Please see <code>bootstrap.example.toml</code> for information on target-specific configuration keys.</p> |
| <p>For examples of the complete configuration necessary to build a target, please visit |
| <a href="https://doc.rust-lang.org/rustc/platform-support.html">the rustc book</a>, |
| select any target under the "Platform Support" heading on the left, |
| and see the section related to building a compiler for that target. |
| For targets without a corresponding page in the rustc book, |
| it may be useful to <a href="building/../tests/docker.html">inspect the Dockerfiles</a> |
| that the Rust infrastructure itself uses to set up and configure cross-compilation.</p> |
| <p>If you have followed the directions from the prior section on creating a rustup toolchain, |
| then once you have built your compiler you will be able to use it to cross-compile like so:</p> |
| <pre><code class="language-bash">cargo +stage1 build --target wasm32-wasip1 |
| </code></pre> |
| <h2 id="other-x-commands"><a class="header" href="#other-x-commands">Other <code>x</code> commands</a></h2> |
| <p>Here are a few other useful <code>x</code> commands. We'll cover some of them in detail |
| in other sections:</p> |
| <ul> |
| <li>Building things: |
| <ul> |
| <li><code>./x build</code> – builds everything using the stage 1 compiler, |
| not just up to <code>std</code></li> |
| <li><code>./x build --stage 2</code> – builds everything with the stage 2 compiler including |
| <code>rustdoc</code></li> |
| </ul> |
| </li> |
| <li>Running tests (see the <a href="building/../tests/running.html">section on running tests</a> for |
| more details): |
| <ul> |
| <li><code>./x test library/std</code> – runs the unit tests and integration tests from <code>std</code></li> |
| <li><code>./x test tests/ui</code> – runs the <code>ui</code> test suite</li> |
| <li><code>./x test tests/ui/const-generics</code> - runs all the tests in |
| the <code>const-generics/</code> subdirectory of the <code>ui</code> test suite</li> |
| <li><code>./x test tests/ui/const-generics/const-types.rs</code> - runs |
| the single test <code>const-types.rs</code> from the <code>ui</code> test suite</li> |
| </ul> |
| </li> |
| </ul> |
| <h3 id="cleaning-out-build-directories"><a class="header" href="#cleaning-out-build-directories">Cleaning out build directories</a></h3> |
| <p>Sometimes you need to start fresh, but this is normally not the case. |
| If you need to run this then bootstrap is most likely not acting right and |
| you should file a bug as to what is going wrong. If you do need to clean |
| everything up then you only need to run one command!</p> |
| <pre><code class="language-bash">./x clean |
| </code></pre> |
| <p><code>rm -rf build</code> works too, but then you have to rebuild LLVM, which can take |
| a long time even on fast computers.</p> |
| <h2 id="remarks-on-disk-space"><a class="header" href="#remarks-on-disk-space">Remarks on disk space</a></h2> |
| <p>Building the compiler (especially if beyond stage 1) can require significant amounts of free disk |
| space, possibly around 100GB. This is compounded if you have a separate build directory for |
| rust-analyzer (e.g. <code>build-rust-analyzer</code>). This is easy to hit with dev-desktops which have a <a href="https://github.com/rust-lang/simpleinfra/blob/8a59e4faeb75a09b072671c74a7cb70160ebef50/ansible/roles/dev-desktop/defaults/main.yml#L7">set |
| disk |
| quota</a> |
| for each user, but this also applies to local development as well. Occasionally, you may need to:</p> |
| <ul> |
| <li>Remove <code>build/</code> directory.</li> |
| <li>Remove <code>build-rust-analyzer/</code> directory (if you have a separate rust-analyzer build directory).</li> |
| <li>Uninstall unnecessary toolchains if you use <code>cargo-bisect-rustc</code>. You can check which toolchains |
| are installed with <code>rustup toolchain list</code>.</li> |
| </ul> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>issue<a href="https://github.com/rust-lang/rustc-dev-guide/issues/1707">#1707</a> <a href="#fr-1-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="quickstart"><a class="header" href="#quickstart">Quickstart</a></h1> |
| <p>This is a quickstart guide about getting the compiler running. For more |
| information on the individual steps, see the other pages in this chapter.</p> |
| <p>First, clone the repository:</p> |
| <pre><code class="language-sh">git clone https://github.com/rust-lang/rust.git |
| cd rust |
| </code></pre> |
| <p>When building the compiler, we don't use <code>cargo</code> directly, instead we use a |
| wrapper called "x". It is invoked with <code>./x</code>.</p> |
| <p>We need to create a configuration for the build. Use <code>./x setup</code> to create a |
| good default.</p> |
| <pre><code class="language-sh">./x setup |
| </code></pre> |
| <p>Then, we can build the compiler. Use <code>./x build</code> to build the compiler, standard |
| library and a few tools. You can also <code>./x check</code> to just check it. All these |
| commands can take specific components/paths as arguments, for example <code>./x check compiler</code> to just check the compiler.</p> |
| <pre><code class="language-sh">./x build |
| </code></pre> |
| <blockquote> |
| <p>When doing a change to the compiler that does not affect the way it compiles |
| the standard library (so for example, a change to an error message), use |
| <code>--keep-stage-std 1</code> to avoid recompiling it.</p> |
| </blockquote> |
| <p>After building the compiler and standard library, you now have a working |
| compiler toolchain. You can use it with rustup by linking it.</p> |
| <pre><code class="language-sh">rustup toolchain link stage1 build/host/stage1 |
| </code></pre> |
| <p>Now you have a toolchain called <code>stage1</code> linked to your build. You can use it to |
| test the compiler.</p> |
| <pre><code class="language-sh">rustc +stage1 testfile.rs |
| </code></pre> |
| <p>After doing a change, you can run the compiler test suite with <code>./x test</code>.</p> |
| <p><code>./x test</code> runs the full test suite, which is slow and rarely what you want. |
| Usually, <code>./x test tests/ui</code> is what you want after a compiler change, testing |
| all <a href="building/../tests/ui.html">UI tests</a> that invoke the compiler on a specific test file |
| and check the output.</p> |
| <pre><code class="language-sh">./x test tests/ui |
| </code></pre> |
| <p>Use <code>--bless</code> if you've made a change and want to update the <code>.stderr</code> files |
| with the new output.</p> |
| <p>Congrats, you are now ready to make a change to the compiler! If you have more |
| questions, <a href="building/./how-to-build-and-run.html">the full chapter</a> might contain the |
| answers, and if it doesn't, feel free to ask for help on |
| <a href="https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp">Zulip</a>.</p> |
| <p>If you use VSCode, Vim, Emacs or Helix, <code>./x setup</code> will ask you if you want to |
| set up the editor config. For more information, check out <a href="building/./suggested.html">suggested |
| workflows</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="prerequisites"><a class="header" href="#prerequisites">Prerequisites</a></h1> |
| <h2 id="dependencies"><a class="header" href="#dependencies">Dependencies</a></h2> |
| <p>See <a href="https://github.com/rust-lang/rust/blob/HEAD/INSTALL.md#dependencies">the <code>rust-lang/rust</code> INSTALL</a>.</p> |
| <h2 id="hardware"><a class="header" href="#hardware">Hardware</a></h2> |
| <p>You will need an internet connection to build. The bootstrapping process |
| involves updating git submodules and downloading a beta compiler. It doesn't |
| need to be super fast, but that can help.</p> |
| <p>There are no strict hardware requirements, but building the compiler is |
| computationally expensive, so a beefier machine will help, and I wouldn't |
| recommend trying to build on a Raspberry Pi! We recommend the following.</p> |
| <ul> |
| <li>30GB+ of free disk space. Otherwise, you will have to keep |
| clearing incremental caches. More space is better, the compiler is a bit of a |
| hog; it's a problem we are aware of.</li> |
| <li>8GB+ RAM</li> |
| <li>2+ cores. Having more cores really helps. 10 or 20 or more is not too many!</li> |
| </ul> |
| <p>Beefier machines will lead to much faster builds. If your machine is not very |
| powerful, a common strategy is to only use <code>./x check</code> on your local machine |
| and let the CI build test your changes when you push to a PR branch.</p> |
| <p>Building the compiler takes more than half an hour on my moderately powerful |
| laptop. We suggest downloading LLVM from CI so you don't have to build it from source |
| (<a href="building/./how-to-build-and-run.html#create-a-bootstraptoml">see here</a>).</p> |
| <p>Like <code>cargo</code>, the build system will use as many cores as possible. Sometimes |
| this can cause you to run low on memory. You can use <code>-j</code> to adjust the number |
| of concurrent jobs. If a full build takes more than ~45 minutes to an hour, you |
| are probably spending most of the time swapping memory in and out; try using |
| <code>-j1</code>.</p> |
| <p>If you don't have too much free disk space, you may want to turn off |
| incremental compilation (<a href="building/./how-to-build-and-run.html#create-a-bootstraptoml">see here</a>). This will make compilation take |
| longer (especially after a rebase), but will save a ton of space from the |
| incremental caches.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="suggested-workflows"><a class="header" href="#suggested-workflows">Suggested workflows</a></h1> |
| <p>The full bootstrapping process takes quite a while. Here are some suggestions to |
| make your life easier.</p> |
| <h2 id="installing-a-pre-push-hook"><a class="header" href="#installing-a-pre-push-hook">Installing a pre-push hook</a></h2> |
| <p>CI will automatically fail your build if it doesn't pass <code>tidy</code>, our internal |
| tool for ensuring code quality. If you'd like, you can install a <a href="https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks">Git |
| hook</a> that will |
| automatically run <code>./x test tidy</code> on each push, to ensure your code is up to |
| par. If the hook fails then run <code>./x test tidy --bless</code> and commit the changes. |
| If you decide later that the pre-push behavior is undesirable, you can delete |
| the <code>pre-push</code> file in <code>.git/hooks</code>.</p> |
| <p>A prebuilt git hook lives at <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/pre-push.sh"><code>src/etc/pre-push.sh</code></a>. It can be copied into |
| your <code>.git/hooks</code> folder as <code>pre-push</code> (without the <code>.sh</code> extension!).</p> |
| <p>You can also install the hook as a step of running <code>./x setup</code>!</p> |
| <h2 id="config-extensions"><a class="header" href="#config-extensions">Config extensions</a></h2> |
| <p>When working on different tasks, you might need to switch between different bootstrap configurations. |
| Sometimes you may want to keep an old configuration for future use. But saving raw config values in |
| random files and manually copying and pasting them can quickly become messy, especially if you have a |
| long history of different configurations.</p> |
| <p>To simplify managing multiple configurations, you can create config extensions.</p> |
| <p>For example, you can create a simple config file named <code>cross.toml</code>:</p> |
| <pre><code class="language-toml">[build] |
| build = "x86_64-unknown-linux-gnu" |
| host = ["i686-unknown-linux-gnu"] |
| target = ["i686-unknown-linux-gnu"] |
| |
| |
| [llvm] |
| download-ci-llvm = false |
| |
| [target.x86_64-unknown-linux-gnu] |
| llvm-config = "/path/to/llvm-19/bin/llvm-config" |
| </code></pre> |
| <p>Then, include this in your <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">include = ["cross.toml"] |
| </code></pre> |
| <p>You can also include extensions within extensions recursively.</p> |
| <p><strong>Note:</strong> In the <code>include</code> field, the overriding logic follows a right-to-left order. For example, |
| in <code>include = ["a.toml", "b.toml"]</code>, extension <code>b.toml</code> overrides <code>a.toml</code>. Also, parent extensions |
| always overrides the inner ones.</p> |
| <h2 id="configuring-rust-analyzer-for-rustc"><a class="header" href="#configuring-rust-analyzer-for-rustc">Configuring <code>rust-analyzer</code> for <code>rustc</code></a></h2> |
| <h3 id="checking-the-library-tree"><a class="header" href="#checking-the-library-tree">Checking the "library" tree</a></h3> |
| <p>Checking the "library" tree requires a stage1 compiler, which can be a heavy process on some computers. |
| For this reason, bootstrap has a flag called <code>--skip-std-check-if-no-download-rustc</code> that skips checking the |
| "library" tree if <code>rust.download-rustc</code> isn't available. If you want to avoid putting a heavy load on your computer |
| with <code>rust-analyzer</code>, you can add the <code>--skip-std-check-if-no-download-rustc</code> flag to your <code>./x check</code> command in |
| the <code>rust-analyzer</code> configuration.</p> |
| <h3 id="project-local-rust-analyzer-setup"><a class="header" href="#project-local-rust-analyzer-setup">Project-local rust-analyzer setup</a></h3> |
| <p><code>rust-analyzer</code> can help you check and format your code whenever you save a |
| file. By default, <code>rust-analyzer</code> runs the <code>cargo check</code> and <code>rustfmt</code> commands, |
| but you can override these commands to use more adapted versions of these tools |
| when hacking on <code>rustc</code>. With custom setup, <code>rust-analyzer</code> can use <code>./x check</code> |
| to check the sources, and the stage 0 rustfmt to format them.</p> |
| <p>The default <code>rust-analyzer.check.overrideCommand</code> command line will check all |
| the crates and tools in the repository. If you are working on a specific part, |
| you can override the command to only check the part you are working on to save |
| checking time. For example, if you are working on the compiler, you can override |
| the command to <code>x check compiler --json-output</code> to only check the compiler part. |
| You can run <code>x check --help --verbose</code> to see the available parts.</p> |
| <p>Running <code>./x setup editor</code> will prompt you to create a project-local LSP config |
| file for one of the supported editors. You can also create the config file as a |
| step of running <code>./x setup</code>.</p> |
| <h3 id="using-a-separate-build-directory-for-rust-analyzer"><a class="header" href="#using-a-separate-build-directory-for-rust-analyzer">Using a separate build directory for rust-analyzer</a></h3> |
| <p>By default, when rust-analyzer runs a check or format command, it will share |
| the same build directory as manual command-line builds. This can be inconvenient |
| for two reasons:</p> |
| <ul> |
| <li>Each build will lock the build directory and force the other to wait, so it |
| becomes impossible to run command-line builds while rust-analyzer is running |
| commands in the background.</li> |
| <li>There is an increased risk of one of the builds deleting previously-built |
| artifacts due to conflicting compiler flags or other settings, forcing |
| additional rebuilds in some cases.</li> |
| </ul> |
| <p>To avoid these problems:</p> |
| <ul> |
| <li>Add <code>--build-dir=build/rust-analyzer</code> to all of the custom <code>x</code> commands in |
| your editor's rust-analyzer configuration. |
| (Feel free to choose a different directory name if desired.)</li> |
| <li>Modify the <code>rust-analyzer.rustfmt.overrideCommand</code> setting so that it points |
| to the copy of <code>rustfmt</code> in that other build directory.</li> |
| <li>Modify the <code>rust-analyzer.procMacro.server</code> setting so that it points to the |
| copy of <code>rust-analyzer-proc-macro-srv</code> in that other build directory.</li> |
| </ul> |
| <p>Using separate build directories for command-line builds and rust-analyzer |
| requires extra disk space.</p> |
| <h3 id="visual-studio-code"><a class="header" href="#visual-studio-code">Visual Studio Code</a></h3> |
| <p>Selecting <code>vscode</code> in <code>./x setup editor</code> will prompt you to create a |
| <code>.vscode/settings.json</code> file which will configure Visual Studio code. The |
| recommended <code>rust-analyzer</code> settings live at |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/rust_analyzer_settings.json"><code>src/etc/rust_analyzer_settings.json</code></a>.</p> |
| <p>If running <code>./x check</code> on save is inconvenient, in VS Code you can use a <a href="https://code.visualstudio.com/docs/editor/tasks">Build |
| Task</a> instead:</p> |
| <pre><code class="language-JSON">// .vscode/tasks.json |
| { |
| "version": "2.0.0", |
| "tasks": [ |
| { |
| "label": "./x check", |
| "command": "./x check", |
| "type": "shell", |
| "problemMatcher": "$rustc", |
| "presentation": { "clear": true }, |
| "group": { "kind": "build", "isDefault": true } |
| } |
| ] |
| } |
| </code></pre> |
| <h3 id="neovim"><a class="header" href="#neovim">Neovim</a></h3> |
| <p>For Neovim users, there are a few options. The |
| easiest way is by using <a href="https://github.com/folke/neoconf.nvim/">neoconf.nvim</a>, |
| which allows for project-local configuration files with the native LSP. The |
| steps for how to use it are below. Note that they require rust-analyzer to |
| already be configured with Neovim. Steps for this can be <a href="https://rust-analyzer.github.io/manual.html#nvim-lsp">found |
| here</a>.</p> |
| <ol> |
| <li>First install the plugin. This can be done by following the steps in the |
| README.</li> |
| <li>Run <code>./x setup editor</code>, and select <code>vscode</code> to create a |
| <code>.vscode/settings.json</code> file. <code>neoconf</code> is able to read and update |
| rust-analyzer settings automatically when the project is opened when this |
| file is detected.</li> |
| </ol> |
| <p>If you're using <code>coc.nvim</code>, you can run <code>./x setup editor</code> and select <code>vim</code> to |
| create a <code>.vim/coc-settings.json</code>. The settings can be edited with |
| <code>:CocLocalConfig</code>. The recommended settings live at |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/rust_analyzer_settings.json"><code>src/etc/rust_analyzer_settings.json</code></a>.</p> |
| <p>Another way is without a plugin, and creating your own logic in your |
| configuration. The following code will work for any checkout of rust-lang/rust (newer than February 2025):</p> |
| <pre><code class="language-lua">local function expand_config_variables(option) |
| local var_placeholders = { |
| ['${workspaceFolder}'] = function(_) |
| return vim.lsp.buf.list_workspace_folders()[1] |
| end, |
| } |
| |
| if type(option) == "table" then |
| local mt = getmetatable(option) |
| local result = {} |
| for k, v in pairs(option) do |
| result[expand_config_variables(k)] = expand_config_variables(v) |
| end |
| return setmetatable(result, mt) |
| end |
| if type(option) ~= "string" then |
| return option |
| end |
| local ret = option |
| for key, fn in pairs(var_placeholders) do |
| ret = ret:gsub(key, fn) |
| end |
| return ret |
| end |
| lspconfig.rust_analyzer.setup { |
| root_dir = function() |
| local default = lspconfig.rust_analyzer.config_def.default_config.root_dir() |
| -- the default root detection uses the cargo workspace root. |
| -- but for rust-lang/rust, the standard library is in its own workspace. |
| -- use the git root instead. |
| local compiler_config = vim.fs.joinpath(default, "../src/bootstrap/defaults/config.compiler.toml") |
| if vim.fs.basename(default) == "library" and vim.uv.fs_stat(compiler_config) then |
| return vim.fs.dirname(default) |
| end |
| return default |
| end, |
| on_init = function(client) |
| local path = client.workspace_folders[1].name |
| local config = vim.fs.joinpath(path, "src/etc/rust_analyzer_zed.json") |
| if vim.uv.fs_stat(config) then |
| -- load rust-lang/rust settings |
| local file = io.open(config) |
| local json = vim.json.decode(file:read("*a")) |
| client.config.settings["rust-analyzer"] = expand_config_variables(json.lsp["rust-analyzer"].initialization_options) |
| client.notify("workspace/didChangeConfiguration", { settings = client.config.settings }) |
| end |
| return true |
| end |
| } |
| </code></pre> |
| <p>If you would like to use the build task that is described above, you may either |
| make your own command in your config, or you can install a plugin such as |
| <a href="https://github.com/stevearc/overseer.nvim">overseer.nvim</a> that can <a href="https://github.com/stevearc/overseer.nvim/blob/master/doc/guides.md#vs-code-tasks">read |
| VSCode's <code>task.json</code> |
| files</a>, |
| and follow the same instructions as above.</p> |
| <h3 id="emacs"><a class="header" href="#emacs">Emacs</a></h3> |
| <p>Emacs provides support for rust-analyzer with project-local configuration |
| through <a href="https://www.gnu.org/software/emacs/manual/html_node/eglot/">Eglot</a>. |
| Steps for setting up Eglot with rust-analyzer can be <a href="https://rust-analyzer.github.io/manual.html#eglot">found |
| here</a>. |
| Having set up Emacs & Eglot for Rust development in general, you can run |
| <code>./x setup editor</code> and select <code>emacs</code>, which will prompt you to create |
| <code>.dir-locals.el</code> with the recommended configuration for Eglot. |
| The recommended settings live at <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/rust_analyzer_eglot.el"><code>src/etc/rust_analyzer_eglot.el</code></a>. |
| For more information on project-specific Eglot configuration, consult <a href="https://www.gnu.org/software/emacs/manual/html_node/eglot/Project_002dspecific-configuration.html">the |
| manual</a>.</p> |
| <h3 id="helix"><a class="header" href="#helix">Helix</a></h3> |
| <p>Helix comes with built-in LSP and rust-analyzer support. |
| It can be configured through <code>languages.toml</code>, as described |
| <a href="https://docs.helix-editor.com/languages.html">here</a>. |
| You can run <code>./x setup editor</code> and select <code>helix</code>, which will prompt you to |
| create <code>languages.toml</code> with the recommended configuration for Helix. The |
| recommended settings live at <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/rust_analyzer_helix.toml"><code>src/etc/rust_analyzer_helix.toml</code></a>.</p> |
| <h3 id="zed"><a class="header" href="#zed">Zed</a></h3> |
| <p>Zed comes with built-in LSP and rust-analyzer support. |
| It can be configured through <code>.zed/settings.json</code>, as described |
| <a href="https://zed.dev/docs/configuring-languages">here</a>. Selecting <code>zed</code> |
| in <code>./x setup editor</code> will prompt you to create a <code>.zed/settings.json</code> |
| file which will configure Zed with the recommended configuration. The |
| recommended <code>rust-analyzer</code> settings live |
| at <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/rust_analyzer_zed.json"><code>src/etc/rust_analyzer_zed.json</code></a>.</p> |
| <h2 id="check-check-and-check-again"><a class="header" href="#check-check-and-check-again">Check, check, and check again</a></h2> |
| <p>When doing simple refactoring, it can be useful to run <code>./x check</code> |
| continuously. If you set up <code>rust-analyzer</code> as described above, this will be |
| done for you every time you save a file. Here you are just checking that the |
| compiler can <strong>build</strong>, but often that is all you need (e.g., when renaming a |
| method). You can then run <code>./x build</code> when you actually need to run tests.</p> |
| <p>In fact, it is sometimes useful to put off tests even when you are not 100% sure |
| the code will work. You can then keep building up refactoring commits and only |
| run the tests at some later time. You can then use <code>git bisect</code> to track down |
| <strong>precisely</strong> which commit caused the problem. A nice side-effect of this style |
| is that you are left with a fairly fine-grained set of commits at the end, all |
| of which build and pass tests. This often helps reviewing.</p> |
| <h2 id="configuring-rustup-to-use-nightly"><a class="header" href="#configuring-rustup-to-use-nightly">Configuring <code>rustup</code> to use nightly</a></h2> |
| <p>Some parts of the bootstrap process uses pinned, nightly versions of tools like |
| rustfmt. To make things like <code>cargo fmt</code> work correctly in your repo, run</p> |
| <pre><code class="language-console">cd <path to rustc repo> |
| rustup override set nightly |
| </code></pre> |
| <p>after <a href="https://rust-lang.github.io/rustup/concepts/channels.html?highlight=nightl#working-with-nightly-rust">installing a nightly toolchain</a> with <code>rustup</code>. Don't forget to do this |
| for all directories you have <a href="building/./suggested.html#working-on-multiple-branches-at-the-same-time">setup a worktree for</a>. You may need to use the |
| pinned nightly version from <code>src/stage0</code>, but often the normal <code>nightly</code> channel |
| will work.</p> |
| <p><strong>Note</strong> see <a href="building/suggested.html#configuring-rust-analyzer-for-rustc">the section on vscode</a> for how to configure it with this real |
| rustfmt <code>x</code> uses, and <a href="building/how-to-build-and-run.html">the section on rustup</a> for how to setup <code>rustup</code> |
| toolchain for your bootstrapped compiler</p> |
| <p><strong>Note</strong> This does <em>not</em> allow you to build <code>rustc</code> with cargo directly. You |
| still have to use <code>x</code> to work on the compiler or standard library, this just |
| lets you use <code>cargo fmt</code>.</p> |
| <h2 id="faster-builds-with-ci-rustc"><a class="header" href="#faster-builds-with-ci-rustc">Faster Builds with CI-rustc</a></h2> |
| <p>If you are not working on the compiler, you often don't need to build the compiler tree. |
| For example, you can skip building the compiler and only build the <code>library</code> tree or the |
| tools under <code>src/tools</code>. To achieve that, you have to enable this by setting the <code>download-rustc</code> |
| option in your configuration. This tells bootstrap to use the latest nightly compiler for <code>stage > 0</code> |
| steps, meaning it will have two precompiled compilers: stage0 compiler and <code>download-rustc</code> compiler |
| for <code>stage > 0</code> steps. This way, it will never need to build the in-tree compiler. As a result, your |
| build time will be significantly reduced by not building the in-tree compiler.</p> |
| <h2 id="faster-rebuilds-with---keep-stage-std"><a class="header" href="#faster-rebuilds-with---keep-stage-std">Faster rebuilds with <code>--keep-stage-std</code></a></h2> |
| <p>Sometimes just checking whether the compiler builds is not enough. A common |
| example is that you need to add a <code>debug!</code> statement to inspect the value of |
| some state or better understand the problem. In that case, you don't really need |
| a full build. By bypassing bootstrap's cache invalidation, you can often get |
| these builds to complete very fast (e.g., around 30 seconds). The only catch is |
| this requires a bit of fudging and may produce compilers that don't work (but |
| that is easily detected and fixed).</p> |
| <p>The sequence of commands you want is as follows:</p> |
| <ul> |
| <li>Initial build: <code>./x build library</code></li> |
| <li>Subsequent builds: <code>./x build library --keep-stage-std=1</code> |
| <ul> |
| <li>Note that we added the <code>--keep-stage-std=1</code> flag here</li> |
| </ul> |
| </li> |
| </ul> |
| <p>As mentioned, the effect of <code>--keep-stage-std=1</code> is that we just <em>assume</em> that the |
| old standard library can be re-used. If you are editing the compiler, this is |
| often true: you haven't changed the standard library, after all. But |
| sometimes, it's not true: for example, if you are editing the "metadata" part of |
| the compiler, which controls how the compiler encodes types and other states |
| into the <code>rlib</code> files, or if you are editing things that wind up in the metadata |
| (such as the definition of the MIR).</p> |
| <p><strong>The TL;DR is that you might get weird behavior from a compile when using |
| <code>--keep-stage-std=1</code></strong> -- for example, strange <a href="building/../appendix/glossary.html#ice">ICEs</a> |
| or other panics. In that case, you should simply remove the <code>--keep-stage-std=1</code> |
| from the command and rebuild. That ought to fix the problem.</p> |
| <p>You can also use <code>--keep-stage-std=1</code> when running tests. Something like this:</p> |
| <ul> |
| <li>Initial test run: <code>./x test tests/ui</code></li> |
| <li>Subsequent test run: <code>./x test tests/ui --keep-stage-std=1</code></li> |
| </ul> |
| <h2 id="using-incremental-compilation"><a class="header" href="#using-incremental-compilation">Using incremental compilation</a></h2> |
| <p>You can further enable the <code>--incremental</code> flag to save additional time in |
| subsequent rebuilds:</p> |
| <pre><code class="language-bash">./x test tests/ui --incremental --test-args issue-1234 |
| </code></pre> |
| <p>If you don't want to include the flag with every command, you can enable it in |
| the <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[rust] |
| incremental = true |
| </code></pre> |
| <p>Note that incremental compilation will use more disk space than usual. If disk |
| space is a concern for you, you might want to check the size of the <code>build</code> |
| directory from time to time.</p> |
| <h2 id="fine-tuning-optimizations"><a class="header" href="#fine-tuning-optimizations">Fine-tuning optimizations</a></h2> |
| <p>Setting <code>optimize = false</code> makes the compiler too slow for tests. However, to |
| improve the test cycle, you can disable optimizations selectively only for the |
| crates you'll have to rebuild |
| (<a href="https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/incremental.20compilation.20question/near/202712165">source</a>). |
| For example, when working on <code>rustc_mir_build</code>, the <code>rustc_mir_build</code> and |
| <code>rustc_driver</code> crates take the most time to incrementally rebuild. You could |
| therefore set the following in the root <code>Cargo.toml</code>:</p> |
| <pre><code class="language-toml">[profile.release.package.rustc_mir_build] |
| opt-level = 0 |
| [profile.release.package.rustc_driver] |
| opt-level = 0 |
| </code></pre> |
| <h2 id="working-on-multiple-branches-at-the-same-time"><a class="header" href="#working-on-multiple-branches-at-the-same-time">Working on multiple branches at the same time</a></h2> |
| <p>Working on multiple branches in parallel can be a little annoying, since |
| building the compiler on one branch will cause the old build and the incremental |
| compilation cache to be overwritten. One solution would be to have multiple |
| clones of the repository, but that would mean storing the Git metadata multiple |
| times, and having to update each clone individually.</p> |
| <p>Fortunately, Git has a better solution called <a href="https://git-scm.com/docs/git-worktree">worktrees</a>. This lets you create |
| multiple "working trees", which all share the same Git database. Moreover, |
| because all of the worktrees share the same object database, if you update a |
| branch (e.g. <code>main</code>) in any of them, you can use the new commits from any of the |
| worktrees. One caveat, though, is that submodules do not get shared. They will |
| still be cloned multiple times.</p> |
| <p>Given you are inside the root directory for your Rust repository, you can create |
| a "linked working tree" in a new "rust2" directory by running the following |
| command:</p> |
| <pre><code class="language-bash">git worktree add ../rust2 |
| </code></pre> |
| <p>Creating a new worktree for a new branch based on <code>main</code> looks like:</p> |
| <pre><code class="language-bash">git worktree add -b my-feature ../rust2 main |
| </code></pre> |
| <p>You can then use that rust2 folder as a separate workspace for modifying and |
| building <code>rustc</code>!</p> |
| <h2 id="working-with-nix"><a class="header" href="#working-with-nix">Working with nix</a></h2> |
| <p>Several nix configurations are defined in <code>src/tools/nix-dev-shell</code>.</p> |
| <p>If you're using direnv, you can create a symbol link to <code>src/tools/nix-dev-shell/envrc-flake</code> or <code>src/tools/nix-dev-shell/envrc-shell</code></p> |
| <pre><code class="language-bash">ln -s ./src/tools/nix-dev-shell/envrc-flake ./.envrc # Use flake |
| </code></pre> |
| <p>or</p> |
| <pre><code class="language-bash">ln -s ./src/tools/nix-dev-shell/envrc-shell ./.envrc # Use nix-shell |
| </code></pre> |
| <h3 id="note"><a class="header" href="#note">Note</a></h3> |
| <p>Note that when using nix on a not-NixOS distribution, it may be necessary to set |
| <strong><code>patch-binaries-for-nix = true</code> in <code>bootstrap.toml</code></strong>. Bootstrap tries to detect |
| whether it's running in nix and enable patching automatically, but this |
| detection can have false negatives.</p> |
| <p>You can also use your nix shell to manage <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-nix">let |
| config = pkgs.writeText "rustc-config" '' |
| # Your bootstrap.toml content goes here |
| '' |
| pkgs.mkShell { |
| /* ... */ |
| # This environment variable tells bootstrap where our bootstrap.toml is. |
| RUST_BOOTSTRAP_CONFIG = config; |
| } |
| </code></pre> |
| <h2 id="shell-completions"><a class="header" href="#shell-completions">Shell Completions</a></h2> |
| <p>If you use Bash, Zsh, Fish or PowerShell, you can find automatically-generated shell |
| completion scripts for <code>x.py</code> in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/etc/completions"><code>src/etc/completions</code></a>.</p> |
| <p>You can use <code>source ./src/etc/completions/x.py.<extension></code> to load completions |
| for your shell of choice, or <code>& .\src\etc\completions\x.py.ps1</code> for PowerShell. |
| Adding this to your shell's startup script (e.g. <code>.bashrc</code>) will automatically |
| load this completion.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="build-distribution-artifacts"><a class="header" href="#build-distribution-artifacts">Build distribution artifacts</a></h1> |
| <p>You might want to build and package up the compiler for distribution. |
| You’ll want to run this command to do it:</p> |
| <pre><code class="language-bash">./x dist |
| </code></pre> |
| <h1 id="install-from-source"><a class="header" href="#install-from-source">Install from source</a></h1> |
| <p>You might want to prefer installing Rust (and tools configured in your configuration) |
| by building from source. If so, you want to run this command:</p> |
| <pre><code class="language-bash">./x install |
| </code></pre> |
| <p>Note: If you are testing out a modification to a compiler, you might |
| want to build the compiler (with <code>./x build</code>) then create a toolchain as |
| discussed in <a href="building/./how-to-build-and-run.html#creating-a-rustup-toolchain">here</a>.</p> |
| <p>For example, if the toolchain you created is called "foo", you would then |
| invoke it with <code>rustc +foo ...</code> (where ... represents the rest of the arguments).</p> |
| <p>Instead of installing Rust (and tools in your config file) globally, you can set <code>DESTDIR</code> |
| environment variable to change the installation path. If you want to set installation paths |
| more dynamically, you should prefer <a href="https://github.com/rust-lang/rust/blob/f7c8928f035370be33463bb7f1cd1aeca2c5f898/config.example.toml#L422-L442">install options</a> in your config file to achieve that.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="building-documentation"><a class="header" href="#building-documentation">Building documentation</a></h1> |
| <p>This chapter describes how to build documentation of toolchain components, |
| like the standard library (std) or the compiler (rustc).</p> |
| <ul> |
| <li> |
| <p>Document everything</p> |
| <p>This uses <code>rustdoc</code> from the beta toolchain, |
| so will produce (slightly) different output to stage 1 rustdoc, |
| as rustdoc is under active development:</p> |
| <pre><code class="language-bash">./x doc |
| </code></pre> |
| <p>If you want to be sure the documentation looks the same as on CI:</p> |
| <pre><code class="language-bash">./x doc --stage 1 |
| </code></pre> |
| <p>This ensures that (current) rustdoc gets built, |
| then that is used to document the components.</p> |
| </li> |
| <li> |
| <p>Much like running individual tests or building specific components, |
| you can build just the documentation you want:</p> |
| <pre><code class="language-bash">./x doc src/doc/book |
| ./x doc src/doc/nomicon |
| ./x doc compiler library |
| </code></pre> |
| <p>See <a href="https://doc.rust-lang.org/nightly/">the nightly docs index page</a> for a full list of books.</p> |
| </li> |
| <li> |
| <p>Document internal rustc items</p> |
| <p>Compiler documentation is not built by default. |
| To create it by default with <code>x doc</code>, modify <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[build] |
| compiler-docs = true |
| </code></pre> |
| <p>Note that when enabled, |
| documentation for internal compiler items will also be built.</p> |
| <p>NOTE: The documentation for the compiler is found at <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/">this link</a>.</p> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rustdoc-overview"><a class="header" href="#rustdoc-overview">Rustdoc overview</a></h1> |
| <p><code>rustdoc</code> lives in-tree with the compiler and standard library. |
| This chapter is about how it works. |
| For information about Rustdoc's features and how to use them, see |
| the <a href="https://doc.rust-lang.org/nightly/rustdoc/">Rustdoc book</a>. |
| For more details about how rustdoc works, see the <a href="./rustdoc-internals.html">"Rustdoc internals" chapter</a>.</p> |
| <p><code>rustdoc</code> uses <code>rustc</code> internals (and, of course, the standard library), so you |
| will have to build the compiler and <code>std</code> once before you can build <code>rustdoc</code>.</p> |
| <p>Rustdoc is implemented entirely within the crate <a href="https://github.com/rust-lang/rust/tree/HEAD/src/librustdoc"><code>librustdoc</code></a>. |
| It runs the compiler up to the point where we have an internal representation of a |
| crate (HIR) and the ability to run some queries about the types of items. |
| <a href="./hir.html">HIR</a> and <a href="./query.html">queries</a> are discussed in the linked chapters.</p> |
| <p><code>librustdoc</code> performs two major steps after that to render a set of documentation:</p> |
| <ul> |
| <li>"Clean" the AST into a form that's more suited to creating documentation (and |
| slightly more resistant to churn in the compiler).</li> |
| <li>Use this cleaned AST to render a crate's documentation, one page at a time.</li> |
| </ul> |
| <p>Naturally, there's more than just this, and those descriptions simplify out |
| lots of details, but that's the high-level overview.</p> |
| <p>(Side note: <code>librustdoc</code> is a library crate! |
| The <code>rustdoc</code> binary is created using the project in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>src/tools/rustdoc</code></a>. |
| Note that literally all that does is call the <code>main()</code> that's in this crate's <code>lib.rs</code>, though.)</p> |
| <h2 id="cheat-sheet"><a class="header" href="#cheat-sheet">Cheat sheet</a></h2> |
| <ul> |
| <li>Run <code>./x setup tools</code> before getting started. This will configure <code>x</code> |
| with nice settings for developing rustdoc and other tools, including |
| downloading a copy of rustc rather than building it.</li> |
| <li>Use <code>./x check rustdoc</code> to quickly check for compile errors.</li> |
| <li>Use <code>./x build library rustdoc</code> to make a usable |
| rustdoc you can run on other projects. |
| <ul> |
| <li>Add <code>library/test</code> to be able to use <code>rustdoc --test</code>.</li> |
| <li>Run <code>rustup toolchain link stage2 build/host/stage2</code> to add a |
| custom toolchain called <code>stage2</code> to your rustup environment. |
| After running that, <code>cargo +stage2 doc</code> in any directory will build with |
| your locally-compiled rustdoc.</li> |
| </ul> |
| </li> |
| <li>Use <code>./x doc library</code> to use this rustdoc to generate the |
| standard library docs. |
| <ul> |
| <li>The completed docs will be available in <code>build/host/doc</code> (under <code>core</code>, <code>alloc</code>, and <code>std</code>).</li> |
| <li>If you want to copy those docs to a webserver, copy all of |
| <code>build/host/doc</code>, since that's where the CSS, JS, fonts, and landing page are.</li> |
| <li>For frontend debugging, disable the <code>rust.docs-minification</code> option in <a href="./building/how-to-build-and-run.html"><code>bootstrap.toml</code></a>.</li> |
| </ul> |
| </li> |
| <li>Use <code>./x test tests/rustdoc*</code> to run the tests using a stage1 |
| rustdoc. |
| <ul> |
| <li>See <a href="./rustdoc-internals.html">Rustdoc internals</a> for more information about tests.</li> |
| </ul> |
| </li> |
| <li>Use <code>./x.py test tidy --extra-checks=js</code> to run rustdoc’s JavaScript checks (<code>eslint</code>, <code>es-check</code>, and <code>tsc</code>).</li> |
| </ul> |
| <blockquote> |
| <p><strong>Note:</strong> <code>./x.py test tidy</code> already runs these checks automatically when JS/TS sources changed; <code>--extra-checks=js</code> forces them explicitly.</p> |
| </blockquote> |
| <h3 id="javascript-ci-checks"><a class="header" href="#javascript-ci-checks">JavaScript CI checks</a></h3> |
| <p>Rustdoc’s JavaScript and TypeScript are checked during CI by <code>eslint</code>, <code>es-check</code>, and <code>tsc</code> (not by compiletest). |
| These run as part of the <code>tidy</code> job.</p> |
| <pre><code class="language-console">./x.py test tidy --extra-checks=js |
| </code></pre> |
| <p>The <code>--extra-checks=js</code> flag enables the frontend linting that runs in CI.</p> |
| <h2 id="code-structure"><a class="header" href="#code-structure">Code structure</a></h2> |
| <p>All paths in this section are relative to <code>src/librustdoc/</code> in the rust-lang/rust repository.</p> |
| <ul> |
| <li>Most of the HTML printing code is in <code>html/format.rs</code> and <code>html/render/mod.rs</code>. |
| It's in a bunch of functions returning <code>impl std::fmt::Display</code>.</li> |
| <li>The data types that get rendered by the functions mentioned above are defined in <code>clean/types.rs</code>. |
| The functions responsible for creating them from the <code>HIR</code> and the <code>rustc_middle::ty</code> IR |
| live in <code>clean/mod.rs</code>.</li> |
| <li>The bits specific to using rustdoc as a test harness are in |
| <code>doctest.rs</code>.</li> |
| <li>The Markdown renderer is loaded up in <code>html/markdown.rs</code>, including functions |
| for extracting doctests from a given block of Markdown.</li> |
| <li>Frontend CSS and JavaScript are stored in <code>html/static/</code>. |
| <ul> |
| <li>Re. JavaScript, type annotations are written using <a href="https://www.typescriptlang.org/docs/handbook/jsdoc-supported-types.html">TypeScript-flavored JSDoc</a> |
| comments and an external <code>.d.ts</code> file. |
| This way, the code itself remains plain, valid JavaScript. |
| We only use <code>tsc</code> as a linter.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="tests"><a class="header" href="#tests">Tests</a></h2> |
| <p><code>rustdoc</code>'s integration tests are split across several test suites. |
| See <a href="tests/compiletest.html#rustdoc-test-suites">Rustdoc tests suites</a> for more details.</p> |
| <h2 id="constraints"><a class="header" href="#constraints">Constraints</a></h2> |
| <p>We try to make rustdoc work reasonably well with JavaScript disabled, and when browsing local files. |
| We have <a href="https://rust-lang.github.io/rfcs/1985-tiered-browser-support.html#supported-browsers">a list of supported browsers</a>.</p> |
| <p>Supporting local files (<code>file:///</code> URLs) brings some surprising restrictions. |
| Certain browser features that require secure origins, like <code>localStorage</code> and |
| Service Workers, don't work reliably. |
| We can still use such features, but we should make sure pages are still usable without them.</p> |
| <p>Rustdoc <a href="https://doc.rust-lang.org/rustdoc/advanced-features.html#interactions-between-platform-specific-docs">does not type-check function bodies</a>. |
| This works by <a href="https://github.com/rust-lang/rust/blob/52bf0cf795dfecc8b929ebb1c1e2545c3f41d4c9/src/librustdoc/core.rs#L299-L323">overriding the built-in queries for typeck</a>, |
| by <a href="https://github.com/rust-lang/rust/blob/52bf0cf795dfecc8b929ebb1c1e2545c3f41d4c9/compiler/rustc_resolve/src/late.rs#L4517">silencing name resolution errors</a>, and by <a href="https://github.com/rust-lang/rust/blob/52bf0cf795dfecc8b929ebb1c1e2545c3f41d4c9/compiler/rustc_hir_analysis/src/check/check.rs#L188-L194">not resolving opaque types</a>. |
| This comes with several caveats: in particular, rustdoc <em>cannot</em> run any parts of the compiler that |
| require type-checking bodies; |
| for example, it cannot generate <code>.rlib</code> files or run most lints. |
| We want to move away from this model eventually, but we need some alternative for |
| <a href="https://github.com/rust-lang/rust/issues/75100">the people using it</a>; |
| see <a href="https://rust-lang.zulipchat.com/#narrow/stream/266220-rustdoc/topic/stop.20accepting.20broken.20code">various</a> |
| <a href="https://rust-lang.zulipchat.com/#narrow/channel/393423-t-rustdoc.2Fmeetings/topic/meeting.202024-07-08/near/449969836">previous</a> <a href="https://rust-lang.zulipchat.com/#narrow/channel/238009-t-compiler.2Fmeetings/topic/.5Bweekly.5D.202023-01-26/near/323755789">Zulip</a> <a href="https://rust-lang.zulipchat.com/#narrow/channel/266220-t-rustdoc/topic/Pre-RFC.3A.20stop.20accepting.20broken.20code">discussions</a>. |
| For examples of code that breaks if this hack is removed, see |
| <a href="https://github.com/rust-lang/rust/tree/163cb4ea3f0ae3bc7921cc259a08a7bf92e73ee6/tests/rustdoc-ui/error-in-impl-trait"><code>tests/rustdoc-ui/error-in-impl-trait</code></a>.</p> |
| <h2 id="multiple-runs-same-output-directory"><a class="header" href="#multiple-runs-same-output-directory">Multiple runs, same output directory</a></h2> |
| <p>Rustdoc can be run multiple times for varying inputs, with its output set to the same directory. |
| That's how cargo produces documentation for dependencies of the current crate. |
| It can also be done manually if a user wants a big |
| documentation bundle with all of the docs they care about.</p> |
| <p>HTML is generated independently for each crate, but there is some cross-crate |
| information that we update as we add crates to the output directory:</p> |
| <ul> |
| <li><code>crates<SUFFIX>.js</code> holds a list of all crates in the output directory.</li> |
| <li><code>search-index<SUFFIX>.js</code> holds a list of all searchable items.</li> |
| <li>For each trait, there is a file under <code>implementors/.../trait.TraitName.js</code> |
| containing a list of implementors of that trait. |
| The implementors may be in |
| different crates than the trait, and the JS file is updated as we discover new ones.</li> |
| </ul> |
| <h2 id="use-cases"><a class="header" href="#use-cases">Use cases</a></h2> |
| <p>There are a few major use cases for rustdoc that you should keep in mind when working on it:</p> |
| <h3 id="standard-library-docs"><a class="header" href="#standard-library-docs">Standard library docs</a></h3> |
| <p>These are published at <a href="https://doc.rust-lang.org/std">https://doc.rust-lang.org/std</a> as part of the Rust release process. |
| Stable releases are also uploaded to specific versioned URLs like |
| <a href="https://doc.rust-lang.org/1.57.0/std/">https://doc.rust-lang.org/1.57.0/std/</a>. |
| Beta and nightly docs are published to |
| <a href="https://doc.rust-lang.org/beta/std/">https://doc.rust-lang.org/beta/std/</a> and <a href="https://doc.rust-lang.org/nightly/std/">https://doc.rust-lang.org/nightly/std/</a>. |
| The docs are uploaded with the <a href="https://github.com/rust-lang/promote-release">promote-release |
| tool</a> and served from S3 with CloudFront.</p> |
| <p>The standard library docs contain five crates: alloc, core, proc_macro, std, and test.</p> |
| <h3 id="docsrs"><a class="header" href="#docsrs">docs.rs</a></h3> |
| <p>When crates are published to crates.io, docs.rs automatically builds |
| and publishes their documentation, for instance at <a href="https://docs.rs/serde/latest/serde/">https://docs.rs/serde/latest/serde/</a>. |
| It always builds with the current nightly |
| rustdoc, so any changes you land in rustdoc are "insta-stable" in that they will |
| have an immediate public effect on docs.rs. |
| Old documentation is only sometimes rebuilt, so |
| you will see some variation in UI when browsing old releases in docs.rs. |
| Crate authors can request rebuilds, which will be run with the latest rustdoc.</p> |
| <p>Docs.rs performs some transformations on rustdoc's output in order to save |
| storage and display a navigation bar at the top. |
| In particular, certain static |
| files, like main.js and rustdoc.css, may be shared across multiple invocations |
| of the same version of rustdoc. |
| Others, like crates.js and sidebar-items.js, are different for different invocations. |
| Still others, like fonts, will never change. |
| These categories are distinguished using the <code>SharedResource</code> enum in |
| <code>src/librustdoc/html/render/write_shared.rs</code></p> |
| <p>Documentation on docs.rs is always generated for a single crate at a time, so |
| the search and sidebar functionality don't include dependencies of the current crate.</p> |
| <h3 id="locally-generated-docs"><a class="header" href="#locally-generated-docs">Locally generated docs</a></h3> |
| <p>Crate authors can run <code>cargo doc --open</code> in crates they have checked out locally to see the docs. |
| This is useful to check that the docs they are writing are useful and display correctly. |
| It can also be useful for people to view documentation on crates they aren't authors of, but want to |
| use. |
| In both cases, people may use <code>--document-private-items</code> Cargo flag to |
| see private methods, fields, and so on, which are normally not displayed.</p> |
| <p>By default, <code>cargo doc</code> will generate documentation for a crate and all of its dependencies. |
| That can result in a very large documentation bundle, with a large (and slow) search corpus. |
| The Cargo flag <code>--no-deps</code> inhibits that behavior and generates docs for just the crate.</p> |
| <h3 id="self-hosted-project-docs"><a class="header" href="#self-hosted-project-docs">Self-hosted project docs</a></h3> |
| <p>Some projects host their own documentation. |
| This is easy to do by locally generating docs, and simply copying them to a web server. |
| Rustdoc's HTML output can be extensively customized by flags. |
| Users can add a theme, set the default theme, and inject arbitrary HTML. |
| See <code>rustdoc --help</code> for details.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="adding-a-new-target"><a class="header" href="#adding-a-new-target">Adding a new target</a></h1> |
| <p>These are a set of steps to add support for a new target. There are |
| numerous end states and paths to get there, so not all sections may be |
| relevant to your desired goal.</p> |
| <p>See also the associated documentation in the <a href="https://doc.rust-lang.org/rustc/target-tier-policy.html#adding-a-new-target">target tier policy</a>.</p> |
| <h2 id="specifying-a-new-llvm"><a class="header" href="#specifying-a-new-llvm">Specifying a new LLVM</a></h2> |
| <p>For very new targets, you may need to use a different fork of LLVM |
| than what is currently shipped with Rust. In that case, navigate to |
| the <code>src/llvm-project</code> git submodule (you might need to run <code>./x check</code> at least once so the submodule is updated), check out the |
| appropriate commit for your fork, then commit that new submodule |
| reference in the main Rust repository.</p> |
| <p>An example would be:</p> |
| <pre><code>cd src/llvm-project |
| git remote add my-target-llvm some-llvm-repository |
| git checkout my-target-llvm/my-branch |
| cd .. |
| git add llvm-project |
| git commit -m 'Use my custom LLVM' |
| </code></pre> |
| <h3 id="using-pre-built-llvm"><a class="header" href="#using-pre-built-llvm">Using pre-built LLVM</a></h3> |
| <p>If you have a local LLVM checkout that is already built, you may be |
| able to configure Rust to treat your build as the system LLVM to avoid |
| redundant builds.</p> |
| <p>You can tell Rust to use a pre-built version of LLVM using the <code>target</code> section |
| of <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[target.x86_64-unknown-linux-gnu] |
| llvm-config = "/path/to/llvm/llvm-7.0.1/bin/llvm-config" |
| </code></pre> |
| <p>If you are attempting to use a system LLVM, we have observed the following paths |
| before, though they may be different from your system:</p> |
| <ul> |
| <li><code>/usr/bin/llvm-config-8</code></li> |
| <li><code>/usr/lib/llvm-8/bin/llvm-config</code></li> |
| </ul> |
| <p>Note that you need to have the LLVM <code>FileCheck</code> tool installed, which is used |
| for codegen tests. This tool is normally built with LLVM, but if you use your |
| own preinstalled LLVM, you will need to provide <code>FileCheck</code> in some other way. |
| On Debian-based systems, you can install the <code>llvm-N-tools</code> package (where <code>N</code> |
| is the LLVM version number, e.g. <code>llvm-8-tools</code>). Alternately, you can specify |
| the path to <code>FileCheck</code> with the <code>llvm-filecheck</code> config item in <code>bootstrap.toml</code> |
| or you can disable codegen test with the <code>codegen-tests</code> item in <code>bootstrap.toml</code>.</p> |
| <h2 id="creating-a-target-specification"><a class="header" href="#creating-a-target-specification">Creating a target specification</a></h2> |
| <p>You should start with a target JSON file. You can see the specification |
| for an existing target using <code>--print target-spec-json</code>:</p> |
| <pre><code>rustc -Z unstable-options --target=wasm32-unknown-unknown --print target-spec-json |
| </code></pre> |
| <p>Save that JSON to a file and modify it as appropriate for your target.</p> |
| <h3 id="adding-a-target-specification"><a class="header" href="#adding-a-target-specification">Adding a target specification</a></h3> |
| <p>Once you have filled out a JSON specification and been able to compile |
| somewhat successfully, you can copy the specification into the |
| compiler itself.</p> |
| <p>You will need to add a line to the big table inside of the |
| <code>supported_targets</code> macro in the <code>rustc_target::spec</code> module. You |
| will then add a corresponding file for your new target containing a |
| <code>target</code> function.</p> |
| <p>Look for existing targets to use as examples.</p> |
| <p>After adding your target to the <code>rustc_target</code> crate you may want to add |
| <code>core</code>, <code>std</code>, ... with support for your new target. In that case you will |
| probably need access to some <code>target_*</code> cfg. Unfortunately when building with |
| stage0 (a precompiled compiler), you'll get an error that the target cfg is |
| unexpected because stage0 doesn't know about the new target specification and |
| we pass <code>--check-cfg</code> in order to tell it to check.</p> |
| <p>To fix the errors you will need to manually add the unexpected value to the |
| different <code>Cargo.toml</code> in <code>library/{std,alloc,core}/Cargo.toml</code>. Here is an |
| example for adding <code>NEW_TARGET_ARCH</code> as <code>target_arch</code>:</p> |
| <p><em><code>library/std/Cargo.toml</code></em>:</p> |
| <pre><code class="language-diff"> [lints.rust.unexpected_cfgs] |
| level = "warn" |
| check-cfg = [ |
| 'cfg(bootstrap)', |
| - 'cfg(target_arch, values("xtensa"))', |
| + # #[cfg(bootstrap)] NEW_TARGET_ARCH |
| + 'cfg(target_arch, values("xtensa", "NEW_TARGET_ARCH"))', |
| </code></pre> |
| <p>To use this target in bootstrap, we need to explicitly add the target triple to the <code>STAGE0_MISSING_TARGETS</code> |
| list in <code>src/bootstrap/src/core/sanity.rs</code>. This is necessary because the default compiler bootstrap uses does |
| not recognize the new target we just added. Therefore, it should be added to <code>STAGE0_MISSING_TARGETS</code> so that the |
| bootstrap is aware that this target is not yet supported by the stage0 compiler.</p> |
| <pre><code class="language-diff">const STAGE0_MISSING_TARGETS: &[&str] = &[ |
| + "NEW_TARGET_TRIPLE" |
| ]; |
| </code></pre> |
| <h2 id="patching-crates"><a class="header" href="#patching-crates">Patching crates</a></h2> |
| <p>You may need to make changes to crates that the compiler depends on, |
| such as <a href="https://crates.io/crates/libc"><code>libc</code></a> or <a href="https://crates.io/crates/cc"><code>cc</code></a>. If so, you can use Cargo's |
| <a href="https://doc.rust-lang.org/stable/cargo/reference/overriding-dependencies.html#the-patch-section"><code>[patch]</code></a> ability. For example, if you want to use an |
| unreleased version of <code>libc</code>, you can add it to the top-level |
| <code>Cargo.toml</code> file:</p> |
| <pre><code class="language-diff">diff --git a/Cargo.toml b/Cargo.toml |
| index 1e83f05e0ca..4d0172071c1 100644 |
| --- a/Cargo.toml |
| +++ b/Cargo.toml |
| @@ -113,6 +113,8 @@ cargo-util = { path = "src/tools/cargo/crates/cargo-util" } |
| [patch.crates-io] |
| +libc = { git = "https://github.com/rust-lang/libc", rev = "0bf7ce340699dcbacabdf5f16a242d2219a49ee0" } |
| |
| # See comments in `src/tools/rustc-workspace-hack/README.md` for what's going on |
| # here |
| rustc-workspace-hack = { path = 'src/tools/rustc-workspace-hack' } |
| </code></pre> |
| <p>After this, run <code>cargo update -p libc</code> to update the lockfiles.</p> |
| <p>Beware that if you patch to a local <code>path</code> dependency, this will enable |
| warnings for that dependency. Some dependencies are not warning-free, and due |
| to the <code>deny-warnings</code> setting in <code>bootstrap.toml</code>, the build may suddenly start |
| to fail. |
| To work around warnings, you may want to:</p> |
| <ul> |
| <li>Modify the dependency to remove the warnings</li> |
| <li>Or for local development purposes, suppress the warnings by setting deny-warnings = false in bootstrap.toml.</li> |
| </ul> |
| <pre><code class="language-toml"># bootstrap.toml |
| [rust] |
| deny-warnings = false |
| </code></pre> |
| <h2 id="cross-compiling"><a class="header" href="#cross-compiling">Cross-compiling</a></h2> |
| <p>Once you have a target specification in JSON and in the code, you can |
| cross-compile <code>rustc</code>:</p> |
| <pre><code>DESTDIR=/path/to/install/in \ |
| ./x install -i --stage 1 --host aarch64-apple-darwin.json --target aarch64-apple-darwin \ |
| compiler/rustc library/std |
| </code></pre> |
| <p>If your target specification is already available in the bootstrap |
| compiler, you can use it instead of the JSON file for both arguments.</p> |
| <h2 id="promoting-a-target-from-tier-2-target-to-tier-2-host"><a class="header" href="#promoting-a-target-from-tier-2-target-to-tier-2-host">Promoting a target from tier 2 (target) to tier 2 (host)</a></h2> |
| <p>There are two levels of tier 2 targets:</p> |
| <ul> |
| <li>Targets that are only cross-compiled (<code>rustup target add</code>)</li> |
| <li>Targets that <a href="https://doc.rust-lang.org/nightly/rustc/target-tier-policy.html#tier-2-with-host-tools">have a native toolchain</a> (<code>rustup toolchain install</code>)</li> |
| </ul> |
| <p>For an example of promoting a target from cross-compiled to native, |
| see <a href="https://github.com/rust-lang/rust/pull/75914">#75914</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="optimized-build-of-the-compiler"><a class="header" href="#optimized-build-of-the-compiler">Optimized build of the compiler</a></h1> |
| <p>There are multiple additional build configuration options and techniques that can be used to compile a |
| build of <code>rustc</code> that is as optimized as possible (for example when building <code>rustc</code> for a Linux |
| distribution). The status of these configuration options for various Rust targets is tracked <a href="https://github.com/rust-lang/rust/issues/103595">here</a>. |
| This page describes how you can use these approaches when building <code>rustc</code> yourself.</p> |
| <h2 id="link-time-optimization"><a class="header" href="#link-time-optimization">Link-time optimization</a></h2> |
| <p>Link-time optimization is a powerful compiler technique that can increase program performance. To |
| enable (Thin-)LTO when building <code>rustc</code>, set the <code>rust.lto</code> config option to <code>"thin"</code> |
| in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[rust] |
| lto = "thin" |
| </code></pre> |
| <blockquote> |
| <p>Note that LTO for <code>rustc</code> is currently supported and tested only for |
| the <code>x86_64-unknown-linux-gnu</code> target. Other targets <em>may</em> work, but no guarantees are provided. |
| Notably, LTO-optimized <code>rustc</code> currently produces <a href="https://github.com/rust-lang/rust/issues/109114">miscompilations</a> on Windows.</p> |
| </blockquote> |
| <p>Enabling LTO on Linux has <a href="https://github.com/rust-lang/rust/pull/101403#issuecomment-1288190019">produced</a> speed-ups by up to 10%.</p> |
| <h2 id="memory-allocator"><a class="header" href="#memory-allocator">Memory allocator</a></h2> |
| <p>Using a different memory allocator for <code>rustc</code> can provide significant performance benefits. If you |
| want to enable the <code>jemalloc</code> allocator, you can set the <code>rust.jemalloc</code> option to <code>true</code> |
| in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[rust] |
| jemalloc = true |
| </code></pre> |
| <blockquote> |
| <p>Note that this option is currently only supported for Linux and macOS targets.</p> |
| </blockquote> |
| <h2 id="codegen-units"><a class="header" href="#codegen-units">Codegen units</a></h2> |
| <p>Reducing the amount of codegen units per <code>rustc</code> crate can produce a faster build of the compiler. |
| You can modify the number of codegen units for <code>rustc</code> and <code>libstd</code> in <code>bootstrap.toml</code> with the |
| following options:</p> |
| <pre><code class="language-toml">[rust] |
| codegen-units = 1 |
| codegen-units-std = 1 |
| </code></pre> |
| <h2 id="instruction-set"><a class="header" href="#instruction-set">Instruction set</a></h2> |
| <p>By default, <code>rustc</code> is compiled for a generic (and conservative) instruction set architecture |
| (depending on the selected target), to make it support as many CPUs as possible. If you want to |
| compile <code>rustc</code> for a specific instruction set architecture, you can set the <code>target_cpu</code> compiler |
| option in <code>RUSTFLAGS</code>:</p> |
| <pre><code class="language-bash">RUSTFLAGS="-C target_cpu=x86-64-v3" ./x build ... |
| </code></pre> |
| <p>If you also want to compile LLVM for a specific instruction set, you can set <code>llvm</code> flags |
| in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[llvm] |
| cxxflags = "-march=x86-64-v3" |
| cflags = "-march=x86-64-v3" |
| </code></pre> |
| <h2 id="profile-guided-optimization"><a class="header" href="#profile-guided-optimization">Profile-guided optimization</a></h2> |
| <p>Applying profile-guided optimizations (or more generally, feedback-directed optimizations) can |
| produce a large increase to <code>rustc</code> performance, by up to 15% (<a href="https://blog.rust-lang.org/inside-rust/2020/11/11/exploring-pgo-for-the-rust-compiler.html#final-numbers-and-a-benchmarking-plot-twist">1</a>, <a href="https://github.com/rust-lang/rust/pull/96978">2</a>). However, these techniques |
| are not simply enabled by a configuration option, but rather they require a complex build workflow |
| that compiles <code>rustc</code> multiple times and profiles it on selected benchmarks.</p> |
| <p>There is a tool called <code>opt-dist</code> that is used to optimize <code>rustc</code> with <a href="https://doc.rust-lang.org/rustc/profile-guided-optimization.html">PGO</a> (profile-guided |
| optimizations) and <a href="https://github.com/llvm/llvm-project/blob/main/bolt/README.md">BOLT</a> (a post-link binary optimizer) for builds distributed to end users. You |
| can examine the tool, which is located in <code>src/tools/opt-dist</code>, and build a custom PGO build |
| workflow based on it, or try to use it directly. Note that the tool is currently quite hardcoded to |
| the way we use it in Rust's continuous integration workflows, and it might require some custom |
| changes to make it work in a different environment.</p> |
| <p>To use the tool, you will need to provide some external dependencies:</p> |
| <ul> |
| <li>A Python3 interpreter (for executing <code>x.py</code>).</li> |
| <li>Compiled LLVM toolchain, with the <code>llvm-profdata</code> binary. Optionally, if you want to use BOLT, |
| the <code>llvm-bolt</code> and |
| <code>merge-fdata</code> binaries have to be available in the toolchain.</li> |
| </ul> |
| <p>These dependencies are provided to <code>opt-dist</code> by an implementation of the <a href="https://github.com/rust-lang/rust/blob/ee451f8faccf3050c76cdcd82543c917b40c7962/src/tools/opt-dist/src/environment.rs#L5"><code>Environment</code></a> struct. |
| It specifies directories where will the PGO/BOLT pipeline take place, and also external dependencies |
| like Python or LLVM.</p> |
| <p>Here is an example of how can <code>opt-dist</code> be used locally (outside of CI):</p> |
| <ol> |
| <li>Enable metrics in your <code>bootstrap.toml</code> file, because <code>opt-dist</code> expects it to be enabled: |
| <pre><code class="language-toml">[build] |
| metrics = true |
| </code></pre> |
| </li> |
| <li>Build the tool with the following command: |
| <pre><code class="language-bash">./x build tools/opt-dist |
| </code></pre> |
| </li> |
| <li>Run the tool with the <code>local</code> mode and provide necessary parameters: |
| <pre><code class="language-bash">./build/host/stage1-tools-bin/opt-dist local \ |
| --target-triple <target> \ # select target, e.g. "x86_64-unknown-linux-gnu" |
| --checkout-dir <path> \ # path to rust checkout, e.g. "." |
| --llvm-dir <path> \ # path to built LLVM toolchain, e.g. "/foo/bar/llvm/install" |
| -- python3 x.py dist # pass the actual build command |
| </code></pre> |
| You can run <code>--help</code> to see further parameters that you can modify.</li> |
| </ol> |
| <blockquote> |
| <p>Note: if you want to run the actual CI pipeline, instead of running <code>opt-dist</code> locally, |
| you can execute <code>cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux</code>.</p> |
| </blockquote> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="testing-the-compiler"><a class="header" href="#testing-the-compiler">Testing the compiler</a></h1> |
| <p>The Rust project runs a wide variety of different tests, orchestrated by the |
| build system (<code>./x test</code>). This section gives a brief overview of the different |
| testing tools. Subsequent chapters dive into <a href="tests/running.html">running tests</a> and |
| <a href="tests/adding.html">adding new tests</a>.</p> |
| <h2 id="kinds-of-tests"><a class="header" href="#kinds-of-tests">Kinds of tests</a></h2> |
| <p>There are several kinds of tests to exercise things in the Rust distribution. |
| Almost all of them are driven by <code>./x test</code>, with some exceptions noted below.</p> |
| <h3 id="compiletest"><a class="header" href="#compiletest">Compiletest</a></h3> |
| <p>The main test harness for testing the compiler itself is a tool called |
| <a href="tests/compiletest.html">compiletest</a>.</p> |
| <p><a href="tests/compiletest.html">compiletest</a> supports running different styles of tests, organized into <em>test |
| suites</em>. A <em>test mode</em> may provide common presets/behavior for a set of <em>test |
| suites</em>. <a href="tests/compiletest.html">compiletest</a>-supported tests are located in the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests"><code>tests</code></a> directory.</p> |
| <p>The <a href="tests/compiletest.html">Compiletest chapter</a> goes into detail on how to use this tool.</p> |
| <blockquote> |
| <p>Example: <code>./x test tests/ui</code></p> |
| </blockquote> |
| <h3 id="package-tests"><a class="header" href="#package-tests">Package tests</a></h3> |
| <p>The standard library and many of the compiler packages include typical Rust |
| <code>#[test]</code> unit tests, integration tests, and documentation tests. You can pass a |
| path to <code>./x test</code> for almost any package in the <code>library/</code> or <code>compiler/</code> |
| directory, and <code>x</code> will essentially run <code>cargo test</code> on that package.</p> |
| <p>Examples:</p> |
| <div class="table-wrapper"><table><thead><tr><th>Command</th><th>Description</th></tr></thead><tbody> |
| <tr><td><code>./x test library/std</code></td><td>Runs tests on <code>std</code> only</td></tr> |
| <tr><td><code>./x test library/core</code></td><td>Runs tests on <code>core</code> only</td></tr> |
| <tr><td><code>./x test compiler/rustc_data_structures</code></td><td>Runs tests on <code>rustc_data_structures</code></td></tr> |
| </tbody></table> |
| </div> |
| <p>The standard library relies very heavily on documentation tests to cover its |
| functionality. However, unit tests and integration tests can also be used as |
| needed. Almost all of the compiler packages have doctests disabled.</p> |
| <p>All standard library and compiler unit tests are placed in separate <code>tests</code> file |
| (which is enforced in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/tools/tidy/src/unit_tests.rs">tidy</a>). This ensures that when the test |
| file is changed, the crate does not need to be recompiled. For example:</p> |
| <pre><code class="language-rust ignore">#[cfg(test)] |
| mod tests;</code></pre> |
| <p>If it wasn't done this way, and you were working on something like <code>core</code>, that |
| would require recompiling the entire standard library, and the entirety of |
| <code>rustc</code>.</p> |
| <p><code>./x test</code> includes some CLI options for controlling the behavior with these |
| package tests:</p> |
| <ul> |
| <li><code>--doc</code> — Only runs documentation tests in the package.</li> |
| <li><code>--no-doc</code> — Run all tests <em>except</em> documentation tests.</li> |
| </ul> |
| <h3 id="tidy"><a class="header" href="#tidy">Tidy</a></h3> |
| <p>Tidy is a custom tool used for validating source code style and formatting |
| conventions, such as rejecting long lines. There is more information in the |
| <a href="tests/../conventions.html#formatting">section on coding conventions</a> or the <a href="https://github.com/rust-lang/rust/blob/HEAD/src/tools/tidy/Readme.md">Tidy Readme</a>.</p> |
| <blockquote> |
| <p>Examples: <code>./x test tidy</code></p> |
| </blockquote> |
| <h3 id="formatting"><a class="header" href="#formatting">Formatting</a></h3> |
| <p>Rustfmt is integrated with the build system to enforce uniform style across the |
| compiler. The formatting check is automatically run by the Tidy tool mentioned |
| above.</p> |
| <p>Examples:</p> |
| <div class="table-wrapper"><table><thead><tr><th>Command</th><th>Description</th></tr></thead><tbody> |
| <tr><td><code>./x fmt --check</code></td><td>Checks formatting and exits with an error if formatting is needed.</td></tr> |
| <tr><td><code>./x fmt</code></td><td>Runs rustfmt across the entire codebase.</td></tr> |
| <tr><td><code>./x test tidy --bless</code></td><td>First runs rustfmt to format the codebase, then runs tidy checks.</td></tr> |
| </tbody></table> |
| </div> |
| <h3 id="book-documentation-tests"><a class="header" href="#book-documentation-tests">Book documentation tests</a></h3> |
| <p>All of the books that are published have their own tests, primarily for |
| validating that the Rust code examples pass. Under the hood, these are |
| essentially using <code>rustdoc --test</code> on the markdown files. The tests can be run |
| by passing a path to a book to <code>./x test</code>.</p> |
| <blockquote> |
| <p>Example: <code>./x test src/doc/book</code></p> |
| </blockquote> |
| <h3 id="documentation-link-checker"><a class="header" href="#documentation-link-checker">Documentation link checker</a></h3> |
| <p>Links across all documentation is validated with a link checker tool, |
| and it can be invoked so:</p> |
| <pre><code class="language-console">./x test linkchecker |
| </code></pre> |
| <p>This requires building all of the documentation, which might take a while.</p> |
| <h3 id="distcheck"><a class="header" href="#distcheck"><code>distcheck</code></a></h3> |
| <p><code>distcheck</code> verifies that the source distribution tarball created by the build |
| system will unpack, build, and run all tests.</p> |
| <pre><code class="language-console">./x test distcheck |
| </code></pre> |
| <h3 id="tool-tests"><a class="header" href="#tool-tests">Tool tests</a></h3> |
| <p>Packages that are included with Rust have all of their tests run as well. This |
| includes things such as cargo, clippy, rustfmt, miri, bootstrap (testing the |
| Rust build system itself), etc.</p> |
| <p>Most of the tools are located in the <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/"><code>src/tools</code></a> directory. To run the tool's |
| tests, just pass its path to <code>./x test</code>.</p> |
| <blockquote> |
| <p>Example: <code>./x test src/tools/cargo</code></p> |
| </blockquote> |
| <p>Usually these tools involve running <code>cargo test</code> within the tool's directory.</p> |
| <p>If you want to run only a specified set of tests, append <code>--test-args FILTER_NAME</code> to the command.</p> |
| <blockquote> |
| <p>Example: <code>./x test src/tools/miri --test-args padding</code></p> |
| </blockquote> |
| <p>In CI, some tools are allowed to fail. Failures send notifications to the |
| corresponding teams, and is tracked on the <a href="https://rust-lang-nursery.github.io/rust-toolstate/">toolstate website</a>. More information |
| can be found in the <a href="https://forge.rust-lang.org/infra/toolstate.html">toolstate documentation</a>.</p> |
| <h3 id="ecosystem-testing"><a class="header" href="#ecosystem-testing">Ecosystem testing</a></h3> |
| <p>Rust tests integration with real-world code to catch regressions and make |
| informed decisions about the evolution of the language. There are several kinds |
| of ecosystem tests, including Crater. See the <a href="tests/ecosystem.html">Ecosystem testing |
| chapter</a> for more details.</p> |
| <h3 id="performance-testing"><a class="header" href="#performance-testing">Performance testing</a></h3> |
| <p>A separate infrastructure is used for testing and tracking performance of the |
| compiler. See the <a href="tests/perf.html">Performance testing chapter</a> for more details.</p> |
| <h3 id="codegen-backend-testing"><a class="header" href="#codegen-backend-testing">Codegen backend testing</a></h3> |
| <p>See <a href="tests/./codegen-backend-tests/intro.html">Codegen backend testing</a>.</p> |
| <h2 id="miscellaneous-information"><a class="header" href="#miscellaneous-information">Miscellaneous information</a></h2> |
| <p>There are some other useful testing-related info at <a href="tests/misc.html">Misc info</a>.</p> |
| <h2 id="further-reading"><a class="header" href="#further-reading">Further reading</a></h2> |
| <p>The following blog posts may also be of interest:</p> |
| <ul> |
| <li>brson's classic <a href="https://brson.github.io/2017/07/10/how-rust-is-tested">"How Rust is tested"</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="running-tests"><a class="header" href="#running-tests">Running tests</a></h1> |
| <p>You can run the entire test collection using <code>x</code>. |
| But note that running the <em>entire</em> test collection is almost never what you want to do during local |
| development because it takes a really long time. |
| For local development, see the subsection after on how to run a subset of tests.</p> |
| <div class="warning"> |
| <p>Running plain <code>./x test</code> will build the stage 1 compiler and then run the whole test suite. |
| This not only includes <code>tests/</code>, but also <code>library/</code>, <code>compiler/</code>, |
| <code>src/tools/</code> package tests and more.</p> |
| <p>You usually only want to run a subset of the test suites (or even a smaller set |
| of tests than that) which you expect will exercise your changes. |
| PR CI exercises a subset of test collections, and merge queue CI will exercise all of the test |
| collection.</p> |
| </div> |
| <pre><code class="language-text">./x test |
| </code></pre> |
| <p>The test results are cached and previously successful tests are <code>ignored</code> during testing. |
| The stdout/stderr contents as well as a timestamp file for every test |
| can be found under <code>build/<target-tuple>/test/</code> for the given |
| <code><target-tuple></code>. To force-rerun a test (e.g. in case the test runner fails to |
| notice a change) you can use the <code>--force-rerun</code> CLI option.</p> |
| <blockquote> |
| <p><strong>Note on requirements of external dependencies</strong></p> |
| <p>Some test suites may require external dependencies. This is especially true of |
| debuginfo tests. Some debuginfo tests require a Python-enabled gdb. You can |
| test if your gdb install supports Python by using the <code>python</code> command from |
| within gdb. Once invoked you can type some Python code (e.g. <code>print("hi")</code>) |
| followed by return and then <code>CTRL+D</code> to execute it. If you are building gdb |
| from source, you will need to configure with |
| <code>--with-python=<path-to-python-binary></code>.</p> |
| </blockquote> |
| <h2 id="running-a-subset-of-the-test-suites"><a class="header" href="#running-a-subset-of-the-test-suites">Running a subset of the test suites</a></h2> |
| <p>When working on a specific PR, you will usually want to run a smaller set of tests. |
| For example, a good "smoke test" that can be used after modifying rustc |
| to see if things are generally working correctly would be to exercise the <code>ui</code> |
| test suite (<a href="https://github.com/rust-lang/rust/tree/HEAD/tests/ui"><code>tests/ui</code></a>):</p> |
| <pre><code class="language-text">./x test tests/ui |
| </code></pre> |
| <p>Of course, the choice of test suites is somewhat arbitrary, and may not suit the task you are doing. |
| For example, if you are hacking on debuginfo, you may be better off with the debuginfo test suite:</p> |
| <pre><code class="language-text">./x test tests/debuginfo |
| </code></pre> |
| <p>If you only need to test a specific subdirectory of tests for any given test |
| suite, you can pass that directory as a filter to <code>./x test</code>:</p> |
| <pre><code class="language-text">./x test tests/ui/const-generics |
| </code></pre> |
| <blockquote> |
| <p><strong>Note for MSYS2</strong></p> |
| <p>On MSYS2 the paths seem to be strange and <code>./x test</code> neither recognizes |
| <code>tests/ui/const-generics</code> nor <code>tests\ui\const-generics</code>. In that case, you can |
| workaround it by using e.g. <code>./x test ui --test-args="tests/ui/const-generics"</code>.</p> |
| </blockquote> |
| <p>Likewise, you can test a single file by passing its path:</p> |
| <pre><code class="language-text">./x test tests/ui/const-generics/const-test.rs |
| </code></pre> |
| <p><code>x</code> doesn't support running a single tool test by passing its path yet. |
| You'll have to use the <code>--test-args</code> argument as described |
| <a href="tests/running.html#running-an-individual-test">below</a>.</p> |
| <pre><code class="language-text">./x test src/tools/miri --test-args tests/fail/uninit/padding-enum.rs |
| </code></pre> |
| <h3 id="run-only-the-tidy-script"><a class="header" href="#run-only-the-tidy-script">Run only the tidy script</a></h3> |
| <pre><code class="language-text">./x test tidy |
| </code></pre> |
| <h3 id="run-tests-on-the-standard-library"><a class="header" href="#run-tests-on-the-standard-library">Run tests on the standard library</a></h3> |
| <pre><code class="language-text">./x test --stage 0 library/std |
| </code></pre> |
| <p>Note that this only runs tests on <code>std</code>; |
| if you want to test <code>core</code> or other crates, you have to specify those explicitly.</p> |
| <h3 id="run-the-tidy-script-and-tests-on-the-standard-library"><a class="header" href="#run-the-tidy-script-and-tests-on-the-standard-library">Run the tidy script and tests on the standard library</a></h3> |
| <pre><code class="language-text">./x test --stage 0 tidy library/std |
| </code></pre> |
| <h3 id="run-tests-on-the-standard-library-using-a-stage-1-compiler"><a class="header" href="#run-tests-on-the-standard-library-using-a-stage-1-compiler">Run tests on the standard library using a stage 1 compiler</a></h3> |
| <pre><code class="language-text">./x test --stage 1 library/std |
| </code></pre> |
| <p>By listing which test suites you want to run, |
| you avoid having to run tests for components you did not change at all.</p> |
| <div class="warning"> |
| <p>Note that bors only runs the tests with the full stage 2 build; |
| therefore, while the tests <strong>usually</strong> work fine with stage 1, there are some limitations.</p> |
| </div> |
| <h3 id="run-all-tests-using-a-stage-2-compiler"><a class="header" href="#run-all-tests-using-a-stage-2-compiler">Run all tests using a stage 2 compiler</a></h3> |
| <pre><code class="language-text">./x test --stage 2 |
| </code></pre> |
| <div class="warning"> |
| You almost never need to do this; |
| CI will run these tests for you. |
| </div> |
| <h2 id="run-unit-tests-on-the-compilerlibrary"><a class="header" href="#run-unit-tests-on-the-compilerlibrary">Run unit tests on the compiler/library</a></h2> |
| <p>You may want to run unit tests on a specific file with following:</p> |
| <pre><code class="language-text">./x test compiler/rustc_data_structures/src/thin_vec/tests.rs |
| </code></pre> |
| <p>But unfortunately, it's impossible. |
| You should invoke the following instead:</p> |
| <pre><code class="language-text">./x test compiler/rustc_data_structures/ --test-args thin_vec |
| </code></pre> |
| <h2 id="running-an-individual-test"><a class="header" href="#running-an-individual-test">Running an individual test</a></h2> |
| <p>Another common thing that people want to do is to run an <strong>individual test</strong>, |
| often the test they are trying to fix. |
| As mentioned earlier, you may pass the |
| full file path to achieve this, or alternatively one may invoke <code>x</code> with the <code>--test-args</code> option:</p> |
| <pre><code class="language-text">./x test tests/ui --test-args issue-1234 |
| </code></pre> |
| <p>Under the hood, the test runner invokes the standard Rust test runner (the same |
| one you get with <code>#[test]</code>), so this command would wind up filtering for tests |
| that include "issue-1234" in the name. |
| Thus, <code>--test-args</code> is a good way to run a collection of related tests.</p> |
| <h2 id="passing-arguments-to-rustc-when-running-tests"><a class="header" href="#passing-arguments-to-rustc-when-running-tests">Passing arguments to <code>rustc</code> when running tests</a></h2> |
| <p>It can sometimes be useful to run some tests with specific compiler arguments, |
| without using <code>RUSTFLAGS</code> (during development of unstable features, with <code>-Z</code> flags, for example).</p> |
| <p>This can be done with <code>./x test</code>'s <code>--compiletest-rustc-args</code> option, to pass |
| additional arguments to the compiler when building the tests.</p> |
| <h2 id="editing-and-updating-the-reference-files"><a class="header" href="#editing-and-updating-the-reference-files">Editing and updating the reference files</a></h2> |
| <p>If you have changed the compiler's output intentionally, or you are making a new |
| test, you can pass <code>--bless</code> to the test subcommand.</p> |
| <p>As an example, if some tests in <code>tests/ui</code> are failing, you can run this command:</p> |
| <pre><code class="language-text">./x test tests/ui --bless |
| </code></pre> |
| <p>It automatically adjusts the <code>.stderr</code>, <code>.stdout</code>, or <code>.fixed</code> files of all <code>test/ui</code> tests. |
| Of course you can also target just specific tests with the <code>--test-args your_test_name</code> flag, |
| just like when running the tests without the <code>--bless</code> flag.</p> |
| <h2 id="configuring-test-running"><a class="header" href="#configuring-test-running">Configuring test running</a></h2> |
| <p>There are a few options for running tests:</p> |
| <ul> |
| <li><code>bootstrap.toml</code> has the <code>rust.verbose-tests</code> option. If <code>false</code>, each test will |
| print a single dot (the default). |
| If <code>true</code>, the name of every test will be printed. |
| This is equivalent to the <code>--quiet</code> option in the <a href="https://doc.rust-lang.org/rustc/tests/">Rust test |
| harness</a>.</li> |
| <li>The environment variable <code>RUST_TEST_THREADS</code> can be set to the number of |
| concurrent threads to use for testing.</li> |
| </ul> |
| <h2 id="passing---pass-mode"><a class="header" href="#passing---pass-mode">Passing <code>--pass $mode</code></a></h2> |
| <p>Pass UI tests now have three modes, <code>check-pass</code>, <code>build-pass</code> and <code>run-pass</code>. |
| When <code>--pass $mode</code> is passed, these tests will be forced to run under the given |
| <code>$mode</code> unless the directive <code>//@ ignore-pass</code> exists in the test file. |
| For example, you can run all the tests in <code>tests/ui</code> as <code>check-pass</code>:</p> |
| <pre><code class="language-text">./x test tests/ui --pass check |
| </code></pre> |
| <p>By passing <code>--pass $mode</code>, you can reduce the testing time. |
| For each mode, please see <a href="tests/ui.html#controlling-passfail-expectations">Controlling pass/fail |
| expectations</a>.</p> |
| <h2 id="running-tests-with-different-compare-modes"><a class="header" href="#running-tests-with-different-compare-modes">Running tests with different "compare modes"</a></h2> |
| <p>UI tests may have different output depending on certain "modes" that the compiler is in. |
| For example, when using the Polonius mode, a test <code>foo.rs</code> will |
| first look for expected output in <code>foo.polonius.stderr</code>, falling back to the |
| usual <code>foo.stderr</code> if not found. |
| The following will run the UI test suite in Polonius mode:</p> |
| <pre><code class="language-text">./x test tests/ui --compare-mode=polonius |
| </code></pre> |
| <p>See <a href="tests/compiletest.html#compare-modes">Compare modes</a> for more details.</p> |
| <h2 id="running-tests-manually"><a class="header" href="#running-tests-manually">Running tests manually</a></h2> |
| <p>Sometimes it's easier and faster to just run the test by hand. |
| Most tests are just <code>.rs</code> files, so after <a href="tests/../building/how-to-build-and-run.html#creating-a-rustup-toolchain">creating a rustup |
| toolchain</a>, you |
| can do something like:</p> |
| <pre><code class="language-text">rustc +stage1 tests/ui/issue-1234.rs |
| </code></pre> |
| <p>This is much faster, but doesn't always work. |
| For example, some tests include |
| directives that specify specific compiler flags, or which rely on other crates, |
| and they may not run the same without those options.</p> |
| <h2 id="running-tests-on-a-remote-machine"><a class="header" href="#running-tests-on-a-remote-machine">Running tests on a remote machine</a></h2> |
| <p>Tests may be run on a remote machine (e.g. to test builds for a different |
| architecture). |
| This is done using <code>remote-test-client</code> on the build machine to |
| send test programs to <code>remote-test-server</code> running on the remote machine. |
| <code>remote-test-server</code> executes the test programs and sends the results back to the build machine. |
| <code>remote-test-server</code> provides <em>unauthenticated remote code |
| execution</em> so be careful where it is used.</p> |
| <p>To do this, first build <code>remote-test-server</code> for the remote machine |
| (using RISC-V as an example):</p> |
| <pre><code class="language-text">./x build src/tools/remote-test-server --target riscv64gc-unknown-linux-gnu |
| </code></pre> |
| <p>The binary will be created at <code>./build/host/stage2-tools/$TARGET_ARCH/release/remote-test-server</code>. |
| Copy this over to the remote machine.</p> |
| <p>On the remote machine, run the <code>remote-test-server</code> with the <code>--bind 0.0.0.0:12345</code> flag (and optionally <code>--verbose</code> flag). |
| Output should look like this:</p> |
| <pre><code class="language-console">$ ./remote-test-server --verbose --bind 0.0.0.0:12345 |
| starting test server |
| listening on 0.0.0.0:12345! |
| </code></pre> |
| <p>Note that binding the server to 0.0.0.0 will allow all hosts able to reach your |
| machine to execute arbitrary code on your machine. |
| We strongly recommend either |
| setting up a firewall to block external access to port 12345, or to use a more |
| restrictive IP address when binding.</p> |
| <p>You can test if the <code>remote-test-server</code> is working by connecting to it and sending <code>ping\n</code>. |
| It should reply <code>pong</code>:</p> |
| <pre><code class="language-console">$ nc $REMOTE_IP 12345 |
| ping |
| pong |
| </code></pre> |
| <p>To run tests using the remote runner, set the <code>TEST_DEVICE_ADDR</code> environment |
| variable then use <code>x</code> as usual. |
| For example, to run <code>ui</code> tests for a RISC-V machine with the IP address <code>1.2.3.4</code> use</p> |
| <pre><code class="language-text">export TEST_DEVICE_ADDR="1.2.3.4:12345" |
| ./x test tests/ui --target riscv64gc-unknown-linux-gnu |
| </code></pre> |
| <p>If <code>remote-test-server</code> was run with the verbose flag, output on the test |
| machine may look something like</p> |
| <pre><code class="language-text">[...] |
| run "/tmp/work/test1007/a" |
| run "/tmp/work/test1008/a" |
| run "/tmp/work/test1009/a" |
| run "/tmp/work/test1010/a" |
| run "/tmp/work/test1011/a" |
| run "/tmp/work/test1012/a" |
| run "/tmp/work/test1013/a" |
| run "/tmp/work/test1014/a" |
| run "/tmp/work/test1015/a" |
| run "/tmp/work/test1016/a" |
| run "/tmp/work/test1017/a" |
| run "/tmp/work/test1018/a" |
| [...] |
| </code></pre> |
| <p>Tests are built on the machine running <code>x</code> not on the remote machine. |
| Tests which fail to build unexpectedly (or <code>ui</code> tests producing incorrect build |
| output) may fail without ever running on the remote machine.</p> |
| <h2 id="testing-on-emulators"><a class="header" href="#testing-on-emulators">Testing on emulators</a></h2> |
| <p>Some platforms are tested via an emulator for architectures that aren't readily available. |
| For architectures where the standard library is well supported and |
| the host operating system supports TCP/IP networking, see the above instructions |
| for testing on a remote machine (in this case the remote machine is emulated).</p> |
| <p>There is also a set of tools for orchestrating running the tests within the emulator. |
| Platforms such as <code>arm-android</code> and <code>arm-unknown-linux-gnueabihf</code> are |
| set up to automatically run the tests under emulation on GitHub Actions. |
| The following will take a look at how a target's tests are run under emulation.</p> |
| <p>The Docker image for <a href="https://github.com/rust-lang/rust/tree/HEAD/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile">armhf-gnu</a> includes <a href="https://www.qemu.org/">QEMU</a> to emulate the ARM CPU architecture. |
| Included in the Rust tree are the tools <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/remote-test-client">remote-test-client</a> and |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/remote-test-server">remote-test-server</a> which are programs for sending test programs and libraries |
| to the emulator, and running the tests within the emulator, and reading the results. |
| The Docker image is set up to launch <code>remote-test-server</code> and the |
| build tools use <code>remote-test-client</code> to communicate with the server to |
| coordinate running tests (see <a href="https://github.com/rust-lang/rust/blob/HEAD/src/bootstrap/src/core/build_steps/test.rs">src/bootstrap/src/core/build_steps/test.rs</a>).</p> |
| <p>To run on the iOS/tvOS/watchOS/visionOS simulator, we can similarly treat it as a "remote" machine. |
| A curious detail here is that the network is shared between |
| the simulator instance and the host macOS, so we can use the local loopback address <code>127.0.0.1</code>. |
| Something like the following should work:</p> |
| <pre><code class="language-sh"># Build the test server for the iOS simulator: |
| ./x build src/tools/remote-test-server --target aarch64-apple-ios-sim |
| |
| # If you already have a simulator instance open, copy the device UUID from: |
| xcrun simctl list devices booted |
| UDID=01234567-89AB-CDEF-0123-456789ABCDEF |
| |
| # Alternatively, create and boot a new simulator instance: |
| xcrun simctl list runtimes |
| xcrun simctl list devicetypes |
| UDID=$(xcrun simctl create $CHOSEN_DEVICE_TYPE $CHOSEN_RUNTIME) |
| xcrun simctl boot $UDID |
| # See https://nshipster.com/simctl/ for details. |
| |
| # Spawn the runner on port 12345: |
| xcrun simctl spawn $UDID ./build/host/stage2-tools/aarch64-apple-ios-sim/release/remote-test-server -v --bind 127.0.0.1:12345 |
| |
| # In a new terminal, run tests via the runner: |
| export TEST_DEVICE_ADDR="127.0.0.1:12345" |
| ./x test --host='' --target aarch64-apple-ios-sim --skip tests/debuginfo |
| # FIXME(madsmtm): Allow debuginfo tests to work (maybe needs `.dSYM` folder to be copied to the target?). |
| </code></pre> |
| <h2 id="testing-tests-on-wasi-wasm32-wasip1"><a class="header" href="#testing-tests-on-wasi-wasm32-wasip1">Testing tests on wasi (wasm32-wasip1)</a></h2> |
| <p>Some tests are specific to wasm targets. |
| To run theste tests, you have to pass <code>--target wasm32-wasip1</code> to <code>x test</code>. |
| Additionally, you need the wasi sdk. |
| Follow the install instructions from the <a href="https://github.com/WebAssembly/wasi-sdk">wasi sdk repository</a> to get a sysroot on your computer. |
| On the <a href="https://github.com/rust-lang/rust/blob/HEAD/src/doc/rustc/src/platform-support/wasm32-wasip1.md#building-the-target.">wasm32-wasip1 target support page</a> a minimum version is specified that your sdk must be able to build. |
| Some cmake commands that take a while and give a lot of very concerning c++ warnings... |
| Then, in <code>bootstrap.toml</code>, point to the sysroot like so:</p> |
| <pre><code class="language-toml">[target.wasm32-wasip1] |
| wasi-root = "<wasi-sdk location>/build/sysroot/install/share/wasi-sysroot" |
| </code></pre> |
| <p>In my case I git-cloned it next to my rust folder, so it was <code>../wasi-sdk/build/....</code> |
| Now, tests should just run, you don't have to set up anything else.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="testing-with-docker"><a class="header" href="#testing-with-docker">Testing with Docker</a></h1> |
| <p>The <a href="https://github.com/rust-lang/rust/tree/HEAD/src/ci/docker"><code>src/ci/docker</code></a> directory includes <a href="https://www.docker.com/">Docker</a> image definitions for Linux-based jobs executed on GitHub Actions (non-Linux jobs run outside Docker). You can run these jobs on your local development machine, which can be |
| helpful to test environments different from your local system. You will |
| need to install Docker on a Linux, Windows, or macOS system (typically Linux |
| will be much faster than Windows or macOS because the latter use virtual |
| machines to emulate a Linux environment).</p> |
| <p>Jobs running in CI are configured through a set of bash scripts, and it is not always trivial to reproduce their behavior locally. If you want to run a CI job locally in the simplest way possible, you can use a provided helper <code>citool</code> that tries to replicate what happens on CI as closely as possible:</p> |
| <pre><code class="language-bash">cargo run --manifest-path src/ci/citool/Cargo.toml run-local <job-name> |
| # For example: |
| cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux-alt |
| </code></pre> |
| <p>If the above script does not work for you, you would like to have more control of the Docker image execution, or you want to understand what exactly happens during Docker job execution, then continue reading below.</p> |
| <h2 id="the-runsh-script"><a class="header" href="#the-runsh-script">The <code>run.sh</code> script</a></h2> |
| <p>The <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/docker/run.sh"><code>src/ci/docker/run.sh</code></a> script is used to build a specific Docker image, run it, |
| build Rust within the image, and either run tests or prepare a set of archives designed for distribution. The script will mount your local Rust source tree in read-only mode, and an <code>obj</code> directory in read-write mode. All the compiler artifacts will be stored in the <code>obj</code> directory. The shell will start out in the <code>obj</code>directory. From there, it will execute <code>../src/ci/run.sh</code> which starts the build as defined by the Docker image.</p> |
| <p>You can run <code>src/ci/docker/run.sh <image-name></code> directly. A few important notes regarding the <code>run.sh</code> script:</p> |
| <ul> |
| <li>When executed on CI, the script expects that all submodules are checked out. If some submodule that is accessed by the job is not available, the build will result in an error. You should thus make sure that you have all required submodules checked out locally. You can either do that manually through git, or set <code>submodules = true</code> in your <code>bootstrap.toml</code> and run a command such as <code>x build</code> to let bootstrap download the most important submodules (this might not be enough for the given CI job that you are trying to execute though).</li> |
| <li><code><image-name></code> corresponds to a single directory located in one of the <code>src/ci/docker/host-*</code> directories. Note that image name does not necessarily correspond to a job name, as some jobs execute the same image, but with different environment variables or Docker build arguments (this is a part of the complexity that makes it difficult to run CI jobs locally).</li> |
| <li>If you are executing a "dist" job (job beginning with <code>dist-</code>), you should set the <code>DEPLOY=1</code> environment variable.</li> |
| <li>If you are executing an "alternative dist" job (job beginning with <code>dist-</code> and ending with <code>-alt</code>), you should set the <code>DEPLOY_ALT=1</code> environment variable.</li> |
| <li>Some of the std tests require IPv6 support. Docker on Linux seems to have it |
| disabled by default. Run the commands in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/scripts/enable-docker-ipv6.sh"><code>enable-docker-ipv6.sh</code></a> to enable |
| IPv6 before creating the container. This only needs to be done once.</li> |
| </ul> |
| <h3 id="interactive-mode"><a class="header" href="#interactive-mode">Interactive mode</a></h3> |
| <p>Sometimes, it can be useful to build a specific Docker image, and then run custom commands inside it, so that you can experiment with how the given system behaves. You can do that using an interactive mode, which will |
| start a bash shell in the container, using <code>src/ci/docker/run.sh --dev <image-name></code>.</p> |
| <p>When inside the Docker container, you can run individual commands to do specific tasks. For |
| example, you can run <code>../x test tests/ui</code> to just run UI tests.</p> |
| <p>Some additional notes about using the interactive mode:</p> |
| <ul> |
| <li>The container will be deleted automatically when you exit the shell, however |
| the build artifacts persist in the <code>obj</code> directory. If you are switching |
| between different Docker images, the artifacts from previous environments |
| stored in the <code>obj</code> directory may confuse the build system. Sometimes you |
| will need to delete parts or all of the <code>obj</code> directory before building |
| inside the container.</li> |
| <li>The container is bare-bones, with only a minimal set of packages. You may |
| want to install some things like <code>apt install less vim</code>.</li> |
| <li>You can open multiple shells in the container. First you need the container |
| name (a short hash), which is displayed in the shell prompt, or you can run |
| <code>docker container ls</code> outside of the container to list the available |
| containers. With the container name, run <code>docker exec -it <CONTAINER> /bin/bash</code> where <code><CONTAINER></code> is the container name like <code>4ba195e95cef</code>.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="testing-with-ci"><a class="header" href="#testing-with-ci">Testing with CI</a></h1> |
| <p>The primary goal of our CI system is to ensure that the <code>main</code> branch of |
| <code>rust-lang/rust</code> is always in a valid state by passing our test suite.</p> |
| <p>From a high-level point of view, when you open a pull request at |
| <code>rust-lang/rust</code>, the following will happen:</p> |
| <ul> |
| <li>A small <a href="tests/ci.html#pull-request-builds">subset</a> of tests and checks are run after each |
| push to the PR. |
| This should help catch common errors.</li> |
| <li>When the PR is approved, the <a href="https://github.com/bors">bors</a> bot enqueues the PR into a <a href="https://bors.rust-lang.org/queue/rust">merge queue</a>.</li> |
| <li>Once the PR gets to the front of the queue, bors will create a merge commit |
| and run the <a href="tests/ci.html#auto-builds">full test suite</a> on it. |
| The merge commit either contains only one specific PR or it can be a <a href="tests/ci.html#rollups">"rollup"</a> which |
| combines multiple PRs together, to reduce CI costs and merge delays.</li> |
| <li>Once the whole test suite finishes, two things can happen. Either CI fails |
| with an error that needs to be addressed by the developer, or CI succeeds and |
| the merge commit is then pushed to the <code>main</code> branch.</li> |
| </ul> |
| <p>If you want to modify what gets executed on CI, see <a href="tests/ci.html#modifying-ci-jobs">Modifying CI jobs</a>.</p> |
| <h2 id="ci-workflow"><a class="header" href="#ci-workflow">CI workflow</a></h2> |
| <!-- date-check: Oct 2024 --> |
| <p>Our CI is primarily executed on <a href="https://github.com/rust-lang/rust/actions">GitHub Actions</a>, with a single workflow defined |
| in <a href="https://github.com/rust-lang/rust/blob/HEAD/.github/workflows/ci.yml"><code>.github/workflows/ci.yml</code></a>, which contains a bunch of steps that are |
| unified for all CI jobs that we execute. |
| When a commit is pushed to a corresponding branch or a PR, the workflow executes the |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/citool"><code>src/ci/citool</code></a> crate, which dynamically generates the specific CI jobs that should be executed. |
| This script uses the <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a> file as an |
| input, which contains a declarative configuration of all our CI jobs.</p> |
| <blockquote> |
| <p>Almost all build steps shell out to separate scripts. This keeps the CI fairly |
| platform independent (i.e., we are not overly reliant on GitHub Actions). |
| GitHub Actions is only relied on for bootstrapping the CI process and for |
| orchestrating the scripts that drive the process.</p> |
| </blockquote> |
| <p>In essence, all CI jobs run <code>./x test</code>, <code>./x dist</code> or some other command with |
| different configurations, across various operating systems, targets, and platforms. |
| There are two broad categories of jobs that are executed, <code>dist</code> and non-<code>dist</code> jobs.</p> |
| <ul> |
| <li>Dist jobs build a full release of the compiler for a specific platform, |
| including all the tools we ship through rustup. |
| Those builds are then uploaded |
| to the <code>rust-lang-ci2</code> S3 bucket and are available to be locally installed |
| with the <a href="https://github.com/kennytm/rustup-toolchain-install-master">rustup-toolchain-install-master</a> tool. |
| The same builds are also used |
| for actual releases: our release process basically consists of copying those |
| artifacts from <code>rust-lang-ci2</code> to the production endpoint and signing them.</li> |
| <li>Non-dist jobs run our full test suite on the platform, and the test suite of |
| all the tools we ship through rustup; |
| The amount of stuff we test depends on |
| the platform (for example some tests are run only on Tier 1 platforms), and |
| some quicker platforms are grouped together on the same builder to avoid wasting CI resources.</li> |
| </ul> |
| <p>Based on an input event (usually a push to a branch), we execute one of three |
| kinds of builds (sets of jobs).</p> |
| <ol> |
| <li>PR builds</li> |
| <li>Auto builds</li> |
| <li>Try builds</li> |
| </ol> |
| <h3 id="pull-request-builds"><a class="header" href="#pull-request-builds">Pull Request builds</a></h3> |
| <p>After each push to a pull request, a set of <code>pr</code> jobs are executed. |
| Currently, these execute the <code>x86_64-gnu-llvm-X</code>, <code>x86_64-gnu-tools</code>, <code>pr-check-1</code>, <code>pr-check-2</code> |
| and <code>tidy</code> jobs, all running on Linux. |
| These execute a relatively short |
| (~40 minutes) and lightweight test suite that should catch common issues. |
| More specifically, they run a set of lints, they try to perform a cross-compile check |
| build to Windows mingw (without producing any artifacts), and they test the |
| compiler using a <em>system</em> version of LLVM. |
| Unfortunately, it would take too many |
| resources to run the full test suite for each commit on every PR.</p> |
| <blockquote> |
| <p><strong>Note on doc comments</strong></p> |
| <p>Note that PR CI as of Oct 2024 <!-- datecheck --> by default does not try to |
| run <code>./x doc xxx</code>. This means that if you have any broken intradoc links that |
| would lead to <code>./x doc xxx</code> failing, it will happen very late into the full |
| merge queue CI pipeline.</p> |
| <p>Thus, it is a good idea to run <code>./x doc xxx</code> locally for any doc comment |
| changes to help catch these early.</p> |
| </blockquote> |
| <p>PR jobs are defined in the <code>pr</code> section of <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a>. |
| Their results can be observed |
| directly on the PR, in the "CI checks" section at the bottom of the PR page.</p> |
| <h3 id="auto-builds"><a class="header" href="#auto-builds">Auto builds</a></h3> |
| <p>Before a commit can be merged into the <code>main</code> branch, it needs to pass our complete test suite. |
| We call this an <code>auto</code> build. |
| This build runs tens of CI jobs that exercise various tests across operating systems and targets. |
| The full test suite is quite slow; |
| it can take several hours until all the <code>auto</code> CI jobs finish.</p> |
| <p>Most platforms only run the build steps, some run a restricted set of tests; |
| only a subset run the full suite of tests (see Rust's <a href="https://forge.rust-lang.org/release/platform-support.html#rust-platform-support">platform tiers</a>).</p> |
| <p>Auto jobs are defined in the <code>auto</code> section of <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a>. |
| They are executed on the <code>auto</code> branch under the <code>rust-lang/rust</code> repository, |
| and the final result will be reported via a comment made by bors on the corresponding PR. |
| The live results can be seen on <a href="https://github.com/rust-lang/rust/actions">the GitHub Actions workflows page</a>.</p> |
| <p>At any given time, at most a single <code>auto</code> build is being executed. |
| Find out more in <a href="tests/ci.html#merging-prs-serially-with-bors">Merging PRs serially with bors</a>.</p> |
| <h3 id="try-builds"><a class="header" href="#try-builds">Try builds</a></h3> |
| <p>Sometimes we want to run a subset of the test suite on CI for a given PR, or |
| build a set of compiler artifacts from that PR, without attempting to merge it. |
| We call this a "try build". |
| A try build is started after a user with the proper |
| permissions posts a PR comment with the <code>@bors try</code> command.</p> |
| <p>There are several use-cases for try builds:</p> |
| <ul> |
| <li>Run a set of performance benchmarks using our <a href="https://github.com/rust-lang/rustc-perf">rustc-perf</a> benchmark suite. |
| For this, a working compiler build is needed, which can be generated with a |
| try build that runs the <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile">dist-x86_64-linux</a> CI job, which builds an optimized |
| version of the compiler on Linux (this job is currently executed by default |
| when you start a try build). |
| To create a try build and schedule it for a |
| performance benchmark, you can use the <code>@bors try @rust-timer queue</code> command combination.</li> |
| <li>Check the impact of the PR across the Rust ecosystem, using a <a href="tests/crater.html">Crater</a> run. |
| Again, a working compiler build is needed for this, which can be produced by |
| the <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile">dist-x86_64-linux</a> CI job.</li> |
| <li>Run a specific CI job (e.g. Windows tests) on a PR, to quickly test if it |
| passes the test suite executed by that job.</li> |
| </ul> |
| <p>By default, if you send a comment with <code>@bors try</code>, the jobs defined in the <code>try</code> section of |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a> will be executed. |
| We call this mode a "fast try build". |
| Such a try build will not execute any tests, and it will allow compilation warnings. |
| It is useful when you want to |
| get an optimized toolchain as fast as possible, for a Crater run or performance benchmarks, |
| even if it might not be working fully correctly. |
| If you want to do a full build for the default try job, |
| specify its job name in a job pattern (explained below).</p> |
| <p>If you want to run custom CI jobs in a try build and make sure that they pass all tests and do |
| not produce any compilation warnings, you can select CI jobs to be executed by specifying a <em>job pattern</em>, |
| which can be used in one of two ways:</p> |
| <ul> |
| <li>You can add a set of <code>try-job: <job pattern></code> directives to the PR description (described below) and then |
| simply run <code>@bors try</code>. |
| CI will read these directives and run the jobs that you have specified. |
| This is |
| useful if you want to rerun the same set of try jobs multiple times, after incrementally modifying a PR.</li> |
| <li>You can specify the job pattern using the <code>jobs</code> parameter of the try command: <code>@bors try jobs=<job pattern></code>. |
| This is useful for one-off try builds with specific jobs. |
| Note that the <code>jobs</code> parameter has a higher priority than the PR description directives. |
| <ul> |
| <li>There can also be multiple patterns specified, e.g. <code>@bors try jobs=job1,job2,job3</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Each job pattern can either be an exact name of a job or a glob pattern that matches multiple jobs, |
| for example <code>*msvc*</code> or <code>*-alt</code>. |
| You can start at most 20 jobs in a single try build. |
| When using |
| glob patterns in the PR description, you can optionally wrap them in backticks (<code>`</code>) to avoid GitHub rendering |
| the pattern as Markdown if it contains e.g. an asterisk. Note that this escaping will not work when using |
| the <code>@bors jobs=</code> parameter.</p> |
| <p>The job pattern needs to match one or more jobs defined in the <code>auto</code> or <code>optional</code> sections |
| of <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a>:</p> |
| <ul> |
| <li><code>auto</code> jobs are executed before a commit is merged into the <code>main</code> branch.</li> |
| <li><code>optional</code> jobs are executed only when explicitly requested via a try build. |
| They are typically used for tier 2 and tier 3 targets.</li> |
| </ul> |
| <p>One reason to do a try build is to do a perf run, as described above, with <code>@rust-timer queue</code>. |
| This perf build then compares against some commit on main. |
| With <code>@bors try parent=<sha></code> you can base your try build and subsequent perf run on a specific commit on <code>main</code>, |
| to help make the perf comparison as fair as possible.</p> |
| <blockquote> |
| <p><strong>Using <code>try-job</code> PR description directives</strong></p> |
| <ol> |
| <li> |
| <p>Identify which set of try-jobs you would like to exercise. You can |
| find the name of the CI jobs in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a>.</p> |
| </li> |
| <li> |
| <p>Amend PR description to include a set of patterns (usually at the end |
| of the PR description), for example:</p> |
| <pre><code class="language-text">This PR fixes #123456. |
| |
| try-job: x86_64-msvc |
| try-job: test-various |
| try-job: `*-alt` |
| </code></pre> |
| <p>Each <code>try-job</code> pattern must be on its own line.</p> |
| </li> |
| <li> |
| <p>Run the prescribed try jobs with <code>@bors try</code>. As aforementioned, this |
| requires the user to either (1) have <code>try</code> permissions or (2) be delegated |
| with <code>try</code> permissions by <code>@bors delegate</code> by someone who has <code>try</code> |
| permissions.</p> |
| </li> |
| </ol> |
| <p>Note that this is usually easier to do than manually edit <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a>. |
| However, it can be less flexible because you cannot adjust the set of tests |
| that are exercised this way.</p> |
| </blockquote> |
| <p>Try builds are executed on the <code>try</code> branch under the <code>rust-lang/rust</code> repository and |
| their results can be seen on <a href="https://github.com/rust-lang/rust/actions">the GitHub Actions workflows page</a>, |
| although usually you will be notified of the result by a comment made by bors on |
| the corresponding PR.</p> |
| <p>Multiple try builds can execute concurrently across different PRs, but there can be at most |
| a single try build running on a single PR at any given time.</p> |
| <p>Note that try builds are handled using the <a href="https://github.com/rust-lang/bors">new bors</a> implementation.</p> |
| <h3 id="modifying-ci-jobs"><a class="header" href="#modifying-ci-jobs">Modifying CI jobs</a></h3> |
| <p>If you want to modify what gets executed on our CI, you can simply modify the |
| <code>pr</code>, <code>auto</code> or <code>try</code> sections of the <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/github-actions/jobs.yml"><code>jobs.yml</code></a> file.</p> |
| <p>You can also modify what gets executed temporarily, for example to test a |
| particular platform or configuration that is challenging to test locally (for |
| example, if a Windows build fails, but you don't have access to a Windows machine). |
| Don't hesitate to use CI resources in such situations.</p> |
| <p>You can perform an arbitrary CI job in two ways:</p> |
| <ul> |
| <li>Use the <a href="tests/ci.html#try-builds">try build</a> functionality, and specify the CI jobs that |
| you want to be executed in try builds in your PR description.</li> |
| <li>Modify the <a href="tests/ci.html#pull-request-builds"><code>pr</code></a> section of <code>jobs.yml</code> to specify which |
| CI jobs should be executed after each push to your PR. |
| This might be faster than repeatedly starting try builds.</li> |
| </ul> |
| <p>To modify the jobs executed after each push to a PR, you can simply copy one of |
| the job definitions from the <code>auto</code> section to the <code>pr</code> section. |
| For example, the <code>x86_64-msvc</code> job is responsible for running the 64-bit MSVC tests. |
| You can copy it to the <code>pr</code> section to cause it to be executed after a commit is pushed |
| to your PR, like this:</p> |
| <pre><code class="language-yaml">pr: |
| ... |
| - image: x86_64-gnu-tools |
| <<: *job-linux-16c |
| # this item was copied from the `auto` section |
| # vvvvvvvvvvvvvvvvvv |
| - image: x86_64-msvc |
| env: |
| RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler |
| SCRIPT: make ci-msvc |
| <<: *job-windows-8c |
| </code></pre> |
| <p>Then you can commit the file and push it to your PR branch on GitHub. |
| GitHub Actions should then execute this CI job after each push to your PR.</p> |
| <div class="warning"> |
| <p><strong>After you have finished your experiments, don't forget to remove any changes |
| you have made to <code>jobs.yml</code>, if they were supposed to be temporary!</strong></p> |
| <p>A good practice is to prefix <code>[WIP]</code> in PR title while still running try jobs |
| and <code>[DO NOT MERGE]</code> in the commit that modifies the CI jobs for testing purposes.</p> |
| </div> |
| <p>Although you are welcome to use CI, just be conscious that this is a shared |
| resource with limited concurrency. |
| Try not to enable too many jobs at once; |
| one or two should be sufficient in most cases.</p> |
| <h2 id="merging-prs-serially-with-bors"><a class="header" href="#merging-prs-serially-with-bors">Merging PRs serially with bors</a></h2> |
| <p>CI services usually test the last commit of a branch merged with the last commit |
| in <code>main</code>, and while that’s great to check if the feature works in isolation, |
| it doesn’t provide any guarantee the code is going to work once it’s merged. |
| Breakages like these usually happen when another, incompatible PR is merged |
| after the build happened.</p> |
| <p>To ensure a <code>main</code> branch that works all the time, we forbid manual merges. |
| Instead, all PRs have to be approved through our bot, <a href="https://github.com/bors">bors</a> (the software |
| behind it is called <a href="https://github.com/rust-lang/homu">homu</a>). |
| All the approved PRs are put in a <a href="https://bors.rust-lang.org/queue/rust">merge queue</a> |
| (sorted by priority and creation date) and are automatically tested one at the time. |
| If all the builders are green, the PR is merged, otherwise the failure is |
| recorded and the PR will have to be re-approved again.</p> |
| <p>Bors doesn’t interact with CI services directly, but it works by pushing the |
| merge commit it wants to test to specific branches (like <code>auto</code> or <code>try</code>), which |
| are configured to execute CI checks. |
| Bors then detects the outcome of the build by listening for either Commit Statuses or Check Runs. |
| Since the merge commit is |
| based on the latest <code>main</code> and only one can be tested at the same time, when |
| the results are green, <code>main</code> is fast-forwarded to that merge commit.</p> |
| <p>Unfortunately, testing a single PR at a time, combined with our long CI (~2 |
| hours for a full run), means we can’t merge a lot of PRs in a single day, and a |
| single failure greatly impacts our throughput. |
| The maximum number of PRs we can merge in a day is around ~10.</p> |
| <p>The long CI run times, and requirement for a large builder pool, is largely due |
| to the fact that full release artifacts are built in the <code>dist-</code> builders. |
| This is worth it because these release artifacts:</p> |
| <ul> |
| <li>Allow perf testing even at a later date.</li> |
| <li>Allow bisection when bugs are discovered later.</li> |
| <li>Ensure release quality since if we're always releasing, we can catch problems |
| early.</li> |
| </ul> |
| <h3 id="rollups"><a class="header" href="#rollups">Rollups</a></h3> |
| <p>Some PRs don’t need the full test suite to be executed: trivial changes like |
| typo fixes or README improvements <em>shouldn’t</em> break the build, and testing every |
| single one of them for 2+ hours would be wasteful. |
| To solve this, we regularly create a "rollup", a PR where we merge several pending trivial PRs so |
| they can be tested together. |
| Rollups are created manually by a team member using |
| the "create a rollup" button on the <a href="https://bors.rust-lang.org/queue/rust">merge queue</a>. |
| The team member uses their judgment to decide if a PR is risky or not.</p> |
| <h2 id="docker"><a class="header" href="#docker">Docker</a></h2> |
| <p>All CI jobs, except those on macOS and Windows, are executed inside that |
| platform’s custom <a href="https://github.com/rust-lang/rust/tree/HEAD/src/ci/docker">Docker container</a>. |
| This has a lot of advantages for us:</p> |
| <ul> |
| <li> |
| <p>The build environment is consistent regardless of the changes of the |
| underlying image (switching from the trusty image to xenial was painless for us).</p> |
| </li> |
| <li> |
| <p>We can use ancient build environments to ensure maximum binary compatibility, |
| for example <a href="https://github.com/rust-lang/rust/blob/HEAD/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile">using older CentOS releases</a> on our Linux builders.</p> |
| </li> |
| <li> |
| <p>We can avoid reinstalling tools (like QEMU or the Android emulator) every time, |
| thanks to Docker image caching.</p> |
| </li> |
| <li> |
| <p>Users can run the same tests in the same environment locally by just running this command:</p> |
| <pre><code>cargo run --manifest-path src/ci/citool/Cargo.toml run-local <job-name> |
| </code></pre> |
| <p>This is helpful for debugging failures. |
| Note that there are only Linux Docker images available locally due to licensing and |
| other restrictions.</p> |
| </li> |
| </ul> |
| <p>The Docker images prefixed with <code>dist-</code> are used for building artifacts while |
| those without that prefix run tests and checks.</p> |
| <p>We also run tests for less common architectures (mainly Tier 2 and Tier 3 platforms) in CI. |
| Since those platforms are not x86, we either run everything |
| inside QEMU, or we just cross-compile if we don’t want to run the tests for that platform.</p> |
| <p>These builders are running on a special pool of builders set up and maintained for us by GitHub.</p> |
| <h2 id="caching"><a class="header" href="#caching">Caching</a></h2> |
| <p>Our CI workflow uses various caching mechanisms, mainly for two things:</p> |
| <h3 id="docker-images-caching"><a class="header" href="#docker-images-caching">Docker images caching</a></h3> |
| <p>The Docker images we use to run most of the Linux-based builders take a <em>long</em> time to fully build. |
| To speed up the build, we cache them using <a href="https://docs.docker.com/build/cache/backends/registry/">Docker registry |
| caching</a>, with the intermediate artifacts being stored on <a href="https://github.com/rust-lang/rust/pkgs/container/rust-ci">ghcr.io</a>. |
| We also push the built Docker images to ghcr, so that they can be reused by other tools |
| (rustup) or by developers running the Docker build locally (to speed up their build).</p> |
| <p>Since we test multiple, diverged branches (<code>main</code>, <code>beta</code> and <code>stable</code>), we |
| can’t rely on a single cache for the images, otherwise builds on a branch would |
| override the cache for the others. |
| Instead, we store the images under different |
| tags, identifying them with a custom hash made from the contents of all the |
| Dockerfiles and related scripts.</p> |
| <p>The CI calculates a hash key, so that the cache of a Docker image is |
| invalidated if one of the following changes:</p> |
| <ul> |
| <li>Dockerfile</li> |
| <li>Files copied into the Docker image in the Dockerfile</li> |
| <li>The architecture of the GitHub runner (x86 or ARM)</li> |
| </ul> |
| <h3 id="llvm-caching-with-sccache"><a class="header" href="#llvm-caching-with-sccache">LLVM caching with Sccache</a></h3> |
| <p>We build some C/C++ stuff in various CI jobs, and we rely on <a href="https://github.com/mozilla/sccache">Sccache</a> to cache |
| the intermediate LLVM artifacts. |
| Sccache is a distributed ccache developed by |
| Mozilla, which can use an object storage bucket as the storage backend.</p> |
| <p>With Sccache there's no need to calculate the hash key ourselves. |
| Sccache invalidates the cache automatically when it detects changes to relevant inputs, |
| such as the source code, the version of the compiler, and important environment variables. |
| So we just pass the Sccache wrapper on top of Cargo and Sccache does the rest.</p> |
| <p>We store the persistent artifacts on the S3 bucket, <code>rust-lang-ci-sccache2</code>. |
| So when the CI runs, if Sccache sees that LLVM is being compiled with the same C/C++ |
| compiler and the LLVM source code is the same, Sccache retrieves the individual |
| compiled translation units from S3.</p> |
| <h2 id="custom-tooling-around-ci"><a class="header" href="#custom-tooling-around-ci">Custom tooling around CI</a></h2> |
| <p>During the years, we developed some custom tooling to improve our CI experience.</p> |
| <h3 id="rust-log-analyzer-to-show-the-error-message-in-prs"><a class="header" href="#rust-log-analyzer-to-show-the-error-message-in-prs">Rust Log Analyzer to show the error message in PRs</a></h3> |
| <p>The build logs for <code>rust-lang/rust</code> are huge, and it’s not practical to find |
| what caused the build to fail by looking at the logs. |
| We therefore developed a bot called <a href="https://github.com/rust-lang/rust-log-analyzer">Rust Log Analyzer</a> (RLA) that |
| receives the build logs on failure, and extracts the error message automatically, |
| posting it on the PR thread.</p> |
| <p>The bot is not hardcoded to look for error strings, but was trained with a bunch |
| of build failures to recognize which lines are common between builds and which are not. |
| While the generated snippets can be weird sometimes, the bot is pretty |
| good at identifying the relevant lines, even if it’s an error we've never seen before.</p> |
| <h3 id="toolstate-to-support-allowed-failures"><a class="header" href="#toolstate-to-support-allowed-failures">Toolstate to support allowed failures</a></h3> |
| <p>The <code>rust-lang/rust</code> repo doesn’t only test the compiler on its CI, but also a |
| variety of tools and documentation. |
| Some documentation is pulled in via git submodules. |
| If we blocked merging rustc PRs on the documentation being fixed, we |
| would be stuck in a chicken-and-egg problem, because the documentation's CI |
| would not pass since updating it would need the not-yet-merged version of rustc |
| to test against (and we usually require CI to be passing).</p> |
| <p>To avoid the problem, submodules are allowed to fail, and their status is |
| recorded in <a href="https://rust-lang-nursery.github.io/rust-toolstate">rust-toolstate</a>. |
| When a submodule breaks, a bot automatically pings |
| the maintainers so they know about the breakage, and it records the failure on |
| the toolstate repository. |
| The release process will then ignore broken tools on |
| nightly, removing them from the shipped nightlies.</p> |
| <p>While tool failures are allowed most of the time, they’re automatically |
| forbidden a week before a release: we don’t care if tools are broken on nightly |
| but they must work on beta and stable, so they also need to work on nightly a |
| few days before we promote nightly to beta.</p> |
| <p>More information is available in the <a href="https://forge.rust-lang.org/infra/toolstate.html">toolstate documentation</a>.</p> |
| <h2 id="public-ci-dashboard"><a class="header" href="#public-ci-dashboard">Public CI dashboard</a></h2> |
| <p>To monitor the Rust CI, you can have a look at the <a href="https://p.datadoghq.com/sb/3a172e20-e9e1-11ed-80e3-da7ad0900002-b5f7bb7e08b664a06b08527da85f7e30">public dashboard</a> maintained by the infra team.</p> |
| <p>These are some useful panels from the dashboard:</p> |
| <ul> |
| <li>Pipeline duration: check how long the auto builds take to run.</li> |
| <li>Top slowest jobs: check which jobs are taking the longest to run.</li> |
| <li>Change in median job duration: check what jobs are slowest than before. Useful |
| to detect regressions.</li> |
| <li>Top failed jobs: check which jobs are failing the most.</li> |
| </ul> |
| <p>To learn more about the dashboard, see the <a href="https://docs.datadoghq.com/continuous_integration/">Datadog CI docs</a>.</p> |
| <h2 id="determining-the-ci-configuration"><a class="header" href="#determining-the-ci-configuration">Determining the CI configuration</a></h2> |
| <p>If you want to determine which <code>bootstrap.toml</code> settings are used in CI for a |
| particular job, it is probably easiest to just look at the build log. |
| To do this:</p> |
| <ol> |
| <li>Go to |
| <a href="https://github.com/rust-lang/rust/actions?query=branch%3Aauto+is%3Asuccess">https://github.com/rust-lang/rust/actions?query=branch%3Aauto+is%3Asuccess</a> |
| to find the most recently successful build, and click on it.</li> |
| <li>Choose the job you are interested in on the left-hand side.</li> |
| <li>Click on the gear icon and choose "View raw logs"</li> |
| <li>Search for the string "Configure the build"</li> |
| <li>All of the build settings are listed on the line with the text, <code>build.configure-args</code></li> |
| </ol> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="adding-new-tests"><a class="header" href="#adding-new-tests">Adding new tests</a></h1> |
| <p><strong>In general, we expect every PR that fixes a bug in rustc to come accompanied |
| by a regression test of some kind.</strong> This test should fail in <code>main</code> but pass |
| after the PR. These tests are really useful for preventing us from repeating the |
| mistakes of the past.</p> |
| <p>The first thing to decide is which kind of test to add. This will depend on the |
| nature of the change and what you want to exercise. Here are some rough |
| guidelines:</p> |
| <ul> |
| <li>The majority of compiler tests are done with <a href="tests/compiletest.html">compiletest</a>. |
| <ul> |
| <li>The majority of compiletest tests are <a href="tests/ui.html">UI</a> tests in the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/ui/"><code>tests/ui</code></a> |
| directory.</li> |
| </ul> |
| </li> |
| <li>Changes to the standard library are usually tested within the standard library |
| itself. |
| <ul> |
| <li>The majority of standard library tests are written as doctests, which |
| illustrate and exercise typical API behavior.</li> |
| <li>Additional <a href="tests/intro.html#package-tests">unit tests</a> should go in |
| <code>library/${crate}/tests</code> (where <code>${crate}</code> is usually <code>core</code>, <code>alloc</code>, or |
| <code>std</code>).</li> |
| </ul> |
| </li> |
| <li>If the code is part of an isolated system, and you are not testing compiler |
| output, consider using a <a href="tests/intro.html#package-tests">unit or integration test</a>.</li> |
| <li>Need to run rustdoc? Prefer a <code>rustdoc</code> or <code>rustdoc-ui</code> test. Occasionally |
| you'll need <code>rustdoc-js</code> as well.</li> |
| <li>Other compiletest test suites are generally used for special purposes: |
| <ul> |
| <li>Need to run gdb or lldb? Use the <code>debuginfo</code> test suite.</li> |
| <li>Need to inspect LLVM IR or MIR IR? Use the <code>codegen</code> or <code>mir-opt</code> test |
| suites.</li> |
| <li>Need to inspect the resulting binary in some way? Or if all the other test |
| suites are too limited for your purposes? Then use <code>run-make</code>. |
| <ul> |
| <li>Use <code>run-make-cargo</code> if you need to exercise in-tree <code>cargo</code> in conjunction |
| with in-tree <code>rustc</code>.</li> |
| </ul> |
| </li> |
| <li>Check out the <a href="tests/compiletest.html">compiletest</a> chapter for more specialized test suites.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>After deciding on which kind of test to add, see <a href="tests/best-practices.html">best |
| practices</a> for guidance on how to author tests that are easy |
| to work with that stand the test of time (i.e. if a test fails or need to be |
| modified several years later, how can we make it easier for them?).</p> |
| <h2 id="ui-test-walkthrough"><a class="header" href="#ui-test-walkthrough">UI test walkthrough</a></h2> |
| <p>The following is a basic guide for creating a <a href="tests/ui.html">UI test</a>, which is one of |
| the most common compiler tests. For this tutorial, we'll be adding a test for an |
| async error message.</p> |
| <h3 id="step-1-add-a-test-file"><a class="header" href="#step-1-add-a-test-file">Step 1: Add a test file</a></h3> |
| <p>The first step is to create a Rust source file somewhere in the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/ui/"><code>tests/ui</code></a> |
| tree. When creating a test, do your best to find a good location and name (see |
| <a href="tests/ui.html#test-organization">Test organization</a> for more). Since naming is the |
| hardest part of development, everything should be downhill from here!</p> |
| <p>Let's place our async test at <code>tests/ui/async-await/await-without-async.rs</code>:</p> |
| <pre><code class="language-rust ignore">// Provide diagnostics when the user writes `await` in a non-`async` function. |
| //@ edition:2018 |
| |
| async fn foo() {} |
| |
| fn bar() { |
| foo().await |
| } |
| |
| fn main() {}</code></pre> |
| <p>A few things to notice about our test:</p> |
| <ul> |
| <li>The top should start with a short comment that <a href="tests/adding.html#explanatory_comment">explains what the test is |
| for</a>.</li> |
| <li>The <code>//@ edition:2018</code> comment is called a <a href="tests/directives.html">directive</a> which |
| provides instructions to compiletest on how to build the test. Here we need to |
| set the edition for <code>async</code> to work (the default is edition 2015).</li> |
| <li>Following that is the source of the test. Try to keep it succinct and to the |
| point. This may require some effort if you are trying to minimize an example |
| from a bug report.</li> |
| <li>We end this test with an empty <code>fn main</code> function. This is because the default |
| for UI tests is a <code>bin</code> crate-type, and we don't want the "main not found" |
| error in our test. Alternatively, you could add <code>#![crate_type="lib"]</code>.</li> |
| </ul> |
| <h3 id="step-2-generate-the-expected-output"><a class="header" href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></h3> |
| <p>The next step is to create the expected output snapshots from the compiler. This |
| can be done with the <code>--bless</code> option:</p> |
| <pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs --bless |
| </code></pre> |
| <p>This will build the compiler (if it hasn't already been built), compile the |
| test, and place the output of the compiler in a file called |
| <code>tests/ui/async-await/await-without-async.stderr</code>.</p> |
| <p>However, this step will fail! You should see an error message, something like |
| this:</p> |
| <blockquote> |
| <p>error: /rust/tests/ui/async-await/await-without-async.rs:7: unexpected |
| error: '7:10: 7:16: <code>await</code> is only allowed inside <code>async</code> functions and |
| blocks E0728'</p> |
| </blockquote> |
| <p>This is because the stderr contains errors which were not matched by error |
| annotations in the source file.</p> |
| <h3 id="step-3-add-error-annotations"><a class="header" href="#step-3-add-error-annotations">Step 3: Add error annotations</a></h3> |
| <p>Every error needs to be annotated with a comment in the source with the text of |
| the error. In this case, we can add the following comment to our test file:</p> |
| <pre><code class="language-rust ignore">fn bar() { |
| foo().await |
| //~^ ERROR `await` is only allowed inside `async` functions and blocks |
| }</code></pre> |
| <p>The <code>//~^</code> squiggle caret comment tells compiletest that the error belongs to |
| the <em>previous</em> line (more on this in the <a href="tests/ui.html#error-annotations">Error |
| annotations</a> section).</p> |
| <p>Save that, and run the test again:</p> |
| <pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs |
| </code></pre> |
| <p>It should now pass, yay!</p> |
| <h3 id="step-4-review-the-output"><a class="header" href="#step-4-review-the-output">Step 4: Review the output</a></h3> |
| <p>Somewhat hand-in-hand with the previous step, you should inspect the <code>.stderr</code> |
| file that was created to see if it looks like how you expect. If you are adding |
| a new diagnostic message, now would be a good time to also consider how readable |
| the message looks overall, particularly for people new to Rust.</p> |
| <p>Our example <code>tests/ui/async-await/await-without-async.stderr</code> file should look |
| like this:</p> |
| <pre><code class="language-text">error[E0728]: `await` is only allowed inside `async` functions and blocks |
| --> $DIR/await-without-async.rs:7:10 |
| | |
| LL | fn bar() { |
| | --- this is not `async` |
| LL | foo().await |
| | ^^^^^^ only allowed inside `async` functions and blocks |
| |
| error: aborting due to previous error |
| |
| For more information about this error, try `rustc --explain E0728`. |
| </code></pre> |
| <p>You may notice some things look a little different than the regular compiler |
| output.</p> |
| <ul> |
| <li>The <code>$DIR</code> removes the path information which will differ between systems.</li> |
| <li>The <code>LL</code> values replace the line numbers. That helps avoid small changes in |
| the source from triggering large diffs. See the |
| <a href="tests/ui.html#normalization">Normalization</a> section for more.</li> |
| </ul> |
| <p>Around this stage, you may need to iterate over the last few steps a few times |
| to tweak your test, re-bless the test, and re-review the output.</p> |
| <h3 id="step-5-check-other-tests"><a class="header" href="#step-5-check-other-tests">Step 5: Check other tests</a></h3> |
| <p>Sometimes when adding or changing a diagnostic message, this will affect other |
| tests in the test suite. The final step before posting a PR is to check if you |
| have affected anything else. Running the UI suite is usually a good start:</p> |
| <pre><code class="language-sh">./x test tests/ui |
| </code></pre> |
| <p>If other tests start failing, you may need to investigate what has changed and |
| if the new output makes sense.</p> |
| <p>You may also need to re-bless the output with the <code>--bless</code> flag.</p> |
| <p><a id="explanatory_comment"></a></p> |
| <h2 id="comment-explaining-what-the-test-is-about"><a class="header" href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></h2> |
| <p>The first comment of a test file should <strong>summarize the point of the test</strong>, and |
| highlight what is important about it. If there is an issue number associated |
| with the test, include the issue number.</p> |
| <p>This comment doesn't have to be super extensive. Just something like "Regression |
| test for #18060: match arms were matching in the wrong order." might already be |
| enough.</p> |
| <p>These comments are very useful to others later on when your test breaks, since |
| they often can highlight what the problem is. They are also useful if for some |
| reason the tests need to be refactored, since they let others know which parts |
| of the test were important. Often a test must be rewritten because it no longer |
| tests what it was meant to test, and then it's useful to know what it <em>was</em> |
| meant to test exactly.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="best-practices-for-writing-tests"><a class="header" href="#best-practices-for-writing-tests">Best practices for writing tests</a></h1> |
| <p>This chapter describes best practices related to authoring and modifying tests. |
| We want to make sure the tests we author are easy to understand and modify, even |
| several years later, without needing to consult the original author and perform |
| a bunch of git archeology.</p> |
| <p>It's good practice to review the test that you authored by pretending that you |
| are a different contributor who is looking at the test that failed several years |
| later without much context (this also helps yourself even a few days or months |
| later!). Then ask yourself: how can I make my life and their lives easier?</p> |
| <p>To help put this into perspective, let's start with an aside on how to write a |
| test that makes the life of another contributor as hard as possible.</p> |
| <blockquote> |
| <p><strong>Aside: Simple Test Sabotage Field Manual</strong></p> |
| <p>To make the life of another contributor as hard as possible, one might:</p> |
| <ul> |
| <li>Name the test after an issue number alone without any other context, e.g. |
| <code>issue-123456.rs</code>.</li> |
| <li>Have no comments at all on what the test is trying to exercise, no links to |
| relevant context.</li> |
| <li>Include a test that is massive (that can otherwise be minimized) and |
| contains non-essential pieces which distracts from the core thing the test |
| is actually trying to test.</li> |
| <li>Include a bunch of unrelated syntax errors and other errors which are not |
| critical to what the test is trying to check.</li> |
| <li>Weirdly format the snippets.</li> |
| <li>Include a bunch of unused and unrelated features.</li> |
| <li>Have e.g. <code>ignore-windows</code> <a href="tests/./directives.html">compiletest directives</a> but don't offer any |
| explanation as to <em>why</em> they are needed.</li> |
| </ul> |
| </blockquote> |
| <h2 id="test-naming"><a class="header" href="#test-naming">Test naming</a></h2> |
| <p>Make it easy for the reader to immediately understand what the test is |
| exercising, instead of having to type in the issue number and dig through github |
| search for what the test is trying to exercise. This has an additional benefit |
| of making the test possible to be filtered via <code>--test-args</code> as a collection of |
| related tests.</p> |
| <ul> |
| <li>Name the test after what it's trying to exercise or prevent regressions of.</li> |
| <li>Keep it concise.</li> |
| <li>Avoid using issue numbers alone as test names.</li> |
| <li>Avoid starting the test name with <code>issue-xxxxx</code> prefix as it degrades |
| auto-completion.</li> |
| </ul> |
| <blockquote> |
| <p><strong>Avoid using only issue numbers as test names</strong></p> |
| <p>Prefer including them as links or <code>#123456</code> in test comments instead. Or if it |
| makes sense to include the issue number, also include brief keywords like |
| <code>macro-external-span-ice-123956.rs</code>.</p> |
| <pre><code class="language-text">tests/ui/typeck/issue-123456.rs // bad |
| tests/ui/typeck/issue-123456-asm-macro-external-span-ice.rs // bad (for tab completion) |
| tests/ui/typeck/asm-macro-external-span-ice-123456.rs // good |
| tests/ui/typeck/asm-macro-external-span-ice.rs // good |
| </code></pre> |
| <p><code>issue-123456.rs</code> does not tell you immediately anything about what the test |
| is actually exercising meaning you need to do additional searching. Including |
| the issue number in the test name as a prefix makes tab completion less useful |
| (if you <code>ls</code> a test directory and get a bunch of <code>issue-xxxxx</code> prefixes). We |
| can link to the issue in a test comment.</p> |
| <pre><code class="language-rs">//! Check that `asm!` macro including nested macros that come from external |
| //! crates do not lead to a codepoint boundary assertion ICE. |
| //! |
| //! Regression test for <https://github.com/rust-lang/rust/issues/123456>. |
| </code></pre> |
| <p>One exception to this rule is <a href="tests/./compiletest.html#crash-tests">crash tests</a>: there it is canonical that |
| tests are named only after issue numbers because its purpose is to track |
| snippets from which issues no longer ICE/crash, and they would either be |
| removed or converted into proper ui/other tests in the fix PRs.</p> |
| </blockquote> |
| <h2 id="test-organization"><a class="header" href="#test-organization">Test organization</a></h2> |
| <ul> |
| <li>For most test suites, try to find a semantically meaningful subdirectory to |
| home the test. |
| <ul> |
| <li>E.g. for an implementation of RFC 2093 specifically, we can group a |
| collection of tests under <code>tests/ui/rfc-2093-infer-outlives/</code>. For the |
| directory name, include what the RFC is about.</li> |
| </ul> |
| </li> |
| <li>For the <a href="tests/./compiletest.html#run-make-tests"><code>run-make</code></a>/<code>run-make-support</code> test suites, each <code>rmake.rs</code> must |
| be contained within an immediate subdirectory under <code>tests/run-make/</code> or |
| <code>tests/run-make-cargo/</code> respectively. Further nesting is not presently |
| supported. Avoid using <em>only</em> an issue number for the test name as well.</li> |
| </ul> |
| <h2 id="test-descriptions"><a class="header" href="#test-descriptions">Test descriptions</a></h2> |
| <p>To help other contributors understand what the test is about if their changes |
| lead to the test failing, we should make sure a test has sufficient docs about |
| its intent/purpose, links to relevant context (incl. issue numbers or other |
| discussions) and possibly relevant resources (e.g. can be helpful to link to |
| Win32 APIs for specific behavior).</p> |
| <p><strong>Synopsis of a test with good comments</strong></p> |
| <pre><code class="language-rust ignore">//! Brief summary of what the test is exercising. |
| //! Example: Regression test for #123456: make sure coverage attribute don't ICE |
| //! when applied to non-items. |
| //! |
| //! Optional: Remarks on related tests/issues, external APIs/tools, crash |
| //! mechanism, how it's fixed, FIXMEs, limitations, etc. |
| //! Example: This test is like `tests/attrs/linkage.rs`, but this test is |
| //! specifically checking `#[coverage]` which exercises a different code |
| //! path. The ICE was triggered during attribute validation when we tried |
| //! to construct a `def_path_str` but only emitted the diagnostic when the |
| //! platform is windows, causing an ICE on unix. |
| //! |
| //! Links to relevant issues and discussions. Examples below: |
| //! Regression test for <https://github.com/rust-lang/rust/issues/123456>. |
| //! See also <https://github.com/rust-lang/rust/issues/101345>. |
| //! See discussion at <https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/123456-example-topic>. |
| //! See [`clone(2)`]. |
| //! |
| //! [`clone(2)`]: https://man7.org/linux/man-pages/man2/clone.2.html |
| |
| //@ ignore-windows |
| // Reason: (why is this test ignored for windows? why not specifically |
| // windows-gnu or windows-msvc?) |
| |
| // Optional: Summary of test cases: What positive cases are checked? |
| // What negative cases are checked? Any specific quirks? |
| |
| fn main() { |
| #[coverage] |
| //~^ ERROR coverage attribute can only be applied to function items. |
| let _ = { |
| // Comment highlighting something that deserves reader attention. |
| fn foo() {} |
| }; |
| }</code></pre> |
| <p>For how much context/explanation is needed, it is up to the author and |
| reviewer's discretion. A good rule of thumb is non-trivial things exercised in |
| the test deserves some explanation to help other contributors to understand. |
| This may include remarks on:</p> |
| <ul> |
| <li>How an ICE can get triggered if it's quite elaborate.</li> |
| <li>Related issues and tests (e.g. this test is like another test but is kept |
| separate because...).</li> |
| <li>Platform-specific behaviors.</li> |
| <li>Behavior of external dependencies and APIs: syscalls, linkers, tools, |
| environments and the likes.</li> |
| </ul> |
| <h2 id="test-content"><a class="header" href="#test-content">Test content</a></h2> |
| <ul> |
| <li>Try to make sure the test is as minimal as possible.</li> |
| <li>Minimize non-critical code and especially minimize unnecessary syntax and type |
| errors which can clutter stderr snapshots.</li> |
| <li>Where possible, use semantically meaningful names (e.g. <code>fn bare_coverage_attributes() {}</code>).</li> |
| </ul> |
| <h2 id="flaky-tests"><a class="header" href="#flaky-tests">Flaky tests</a></h2> |
| <p>All tests need to strive to be reproducible and reliable. Flaky tests are the |
| worst kind of tests, arguably even worse than not having the test in the first |
| place.</p> |
| <ul> |
| <li>Flaky tests can fail in completely unrelated PRs which can confuse other |
| contributors and waste their time trying to figure out if test failure is |
| related.</li> |
| <li>Flaky tests provide no useful information from its test results other than |
| it's flaky and not reliable: if a test passed but it's flakey, did I just get |
| lucky? if a test is flakey but it failed, was it just spurious?</li> |
| <li>Flaky tests degrade confidence in the whole test suite. If a test suite can |
| randomly spuriously fail due to flaky tests, did the whole test suite pass or |
| did I just get lucky/unlucky?</li> |
| <li>Flaky tests can randomly fail in full CI, wasting previous full CI resources.</li> |
| </ul> |
| <h2 id="compiletest-directives"><a class="header" href="#compiletest-directives">Compiletest directives</a></h2> |
| <p>See <a href="tests/./directives.html">compiletest directives</a> for a listing of directives.</p> |
| <ul> |
| <li>For <code>ignore-*</code>/<code>needs-*</code>/<code>only-*</code> directives, unless extremely obvious, |
| provide a brief remark on why the directive is needed. E.g. <code>"//@ ignore-wasi (wasi codegens the main symbol differently)"</code>.</li> |
| <li>When using <code>//@ ignore-auxiliary</code>, specify the corresponding main test files, |
| e.g. <code>//@ ignore-auxiliary (used by `./foo.rs`)</code>.</li> |
| </ul> |
| <h2 id="filecheck-best-practices"><a class="header" href="#filecheck-best-practices">FileCheck best practices</a></h2> |
| <p>See <a href="https://llvm.org/docs/CommandGuide/FileCheck.html">LLVM FileCheck guide</a> for details.</p> |
| <ul> |
| <li>Avoid matching on specific register numbers or basic block numbers unless |
| they're special or critical for the test. Consider using patterns to match |
| them where suitable.</li> |
| </ul> |
| <blockquote> |
| <p><strong>TODO</strong></p> |
| <p>Pending concrete advice.</p> |
| </blockquote> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="compiletest-1"><a class="header" href="#compiletest-1">Compiletest</a></h1> |
| <h2 id="introduction"><a class="header" href="#introduction">Introduction</a></h2> |
| <p><code>compiletest</code> is the main test harness of the Rust test suite. It allows test |
| authors to organize large numbers of tests (the Rust compiler has many |
| thousands), efficient test execution (parallel execution is supported), and |
| allows the test author to configure behavior and expected results of both |
| individual and groups of tests.</p> |
| <blockquote> |
| <p><strong>Note for macOS users</strong></p> |
| <p>For macOS users, <code>SIP</code> (System Integrity Protection) <a href="https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp/topic/.E2.9C.94.20Is.20there.20any.20performance.20issue.20for.20MacOS.3F">may consistently check |
| the compiled binary by sending network requests to Apple</a>, so you may |
| get a huge performance degradation when running tests.</p> |
| <p>You can resolve it by tweaking the following settings: <code>Privacy & Security -> Developer Tools -> Add Terminal (Or VsCode, etc.)</code>.</p> |
| </blockquote> |
| <p><code>compiletest</code> may check test code for compile-time or run-time success/failure.</p> |
| <p>Tests are typically organized as a Rust source file with annotations in comments |
| before and/or within the test code. These comments serve to direct <code>compiletest</code> |
| on if or how to run the test, what behavior to expect, and more. See |
| <a href="tests/directives.html">directives</a> and the test suite documentation below for more details |
| on these annotations.</p> |
| <p>See the <a href="tests/adding.html">Adding new tests</a> and <a href="tests/best-practices.html">Best practices</a> |
| chapters for a tutorial on creating a new test and advice on writing a good |
| test, and the <a href="tests/running.html">Running tests</a> chapter on how to run the test suite.</p> |
| <p>Arguments can be passed to compiletest using <code>--test-args</code> or by placing them after <code>--</code>, e.g.</p> |
| <ul> |
| <li><code>x test --test-args --force-rerun</code></li> |
| <li><code>x test -- --force-rerun</code></li> |
| </ul> |
| <p>Additionally, bootstrap accepts several common arguments directly, e.g.</p> |
| <p><code>x test --no-capture --force-rerun --run --pass</code>.</p> |
| <p>Compiletest itself tries to avoid running tests when the artifacts that are |
| involved (mainly the compiler) haven't changed. You can use <code>x test --test-args --force-rerun</code> to rerun a test even when none of the inputs have changed.</p> |
| <h2 id="test-suites"><a class="header" href="#test-suites">Test suites</a></h2> |
| <p>All of the tests are in the <a href="https://github.com/rust-lang/rust/blob/HEAD/tests"><code>tests</code></a> directory. The tests are organized into |
| "suites", with each suite in a separate subdirectory. Each test suite behaves a |
| little differently, with different compiler behavior and different checks for |
| correctness. For example, the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/incremental"><code>tests/incremental</code></a> directory contains tests for |
| incremental compilation. The various suites are defined in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/common.rs"><code>src/tools/compiletest/src/common.rs</code></a> in the <code>pub enum Mode</code> declaration.</p> |
| <p>The following test suites are available, with links for more information:</p> |
| <h3 id="compiler-specific-test-suites"><a class="header" href="#compiler-specific-test-suites">Compiler-specific test suites</a></h3> |
| <div class="table-wrapper"><table><thead><tr><th>Test suite</th><th>Purpose</th></tr></thead><tbody> |
| <tr><td><a href="tests/ui.html"><code>ui</code></a></td><td>Check the stdout/stderr snapshots from the compilation and/or running the resulting executable</td></tr> |
| <tr><td><code>ui-fulldeps</code></td><td><code>ui</code> tests which require a linkable build of <code>rustc</code> (such as using <code>extern crate rustc_span;</code> or used as a plugin)</td></tr> |
| <tr><td><a href="tests/compiletest.html#pretty-printer-tests"><code>pretty</code></a></td><td>Check pretty printing</td></tr> |
| <tr><td><a href="tests/compiletest.html#incremental-tests"><code>incremental</code></a></td><td>Check incremental compilation behavior</td></tr> |
| <tr><td><a href="tests/compiletest.html#debuginfo-tests"><code>debuginfo</code></a></td><td>Check debuginfo generation running debuggers</td></tr> |
| <tr><td><a href="tests/compiletest.html#codegen-tests"><code>codegen-*</code></a></td><td>Check code generation</td></tr> |
| <tr><td><a href="tests/compiletest.html#codegen-units-tests"><code>codegen-units</code></a></td><td>Check codegen unit partitioning</td></tr> |
| <tr><td><a href="tests/compiletest.html#assembly-tests"><code>assembly</code></a></td><td>Check assembly output</td></tr> |
| <tr><td><a href="tests/compiletest.html#mir-opt-tests"><code>mir-opt</code></a></td><td>Check MIR generation and optimizations</td></tr> |
| <tr><td><a href="tests/compiletest.html#coverage-tests"><code>coverage</code></a></td><td>Check coverage instrumentation</td></tr> |
| <tr><td><a href="tests/compiletest.html#coverage-tests"><code>coverage-run-rustdoc</code></a></td><td><code>coverage</code> tests that also run instrumented doctests</td></tr> |
| <tr><td><a href="tests/compiletest.html#crash-tests"><code>crashes</code></a></td><td>Check that the compiler ICEs/panics/crashes on certain inputs to catch accidental fixes</td></tr> |
| </tbody></table> |
| </div> |
| <h3 id="general-purpose-test-suite"><a class="header" href="#general-purpose-test-suite">General purpose test suite</a></h3> |
| <p><a href="tests/compiletest.html#run-make-tests"><code>run-make</code></a> are general purpose tests using Rust programs.</p> |
| <h3 id="rustdoc-test-suites"><a class="header" href="#rustdoc-test-suites">Rustdoc test suites</a></h3> |
| <div class="table-wrapper"><table><thead><tr><th>Test suite</th><th>Purpose</th></tr></thead><tbody> |
| <tr><td><a href="tests/../rustdoc-internals/rustdoc-test-suite.html"><code>rustdoc</code></a></td><td>Check HTML output of <code>rustdoc</code></td></tr> |
| <tr><td><a href="tests/../rustdoc-internals/rustdoc-gui-test-suite.html"><code>rustdoc-gui</code></a></td><td>Check <code>rustdoc</code>'s GUI using a web browser</td></tr> |
| <tr><td><a href="tests/../rustdoc-internals/search.html#testing-the-search-engine"><code>rustdoc-js</code></a></td><td>Check <code>rustdoc</code>'s search engine and index</td></tr> |
| <tr><td><a href="tests/../rustdoc-internals/search.html#testing-the-search-engine"><code>rustdoc-js-std</code></a></td><td>Check <code>rustdoc</code>'s search engine and index on the std library docs</td></tr> |
| <tr><td><a href="tests/../rustdoc-internals/rustdoc-json-test-suite.html"><code>rustdoc-json</code></a></td><td>Check JSON output of <code>rustdoc</code></td></tr> |
| <tr><td><code>rustdoc-ui</code></td><td>Check terminal output of <code>rustdoc</code> (<a href="tests/ui.html">see also</a>)</td></tr> |
| </tbody></table> |
| </div> |
| <p>Some rustdoc-specific tests can also be found in <code>ui/rustdoc/</code>. |
| These check rustdoc-related or -specific lints that (also) run as part of <code>rustc</code>, not (only) <code>rustdoc</code>. |
| Run-make tests pertaining to rustdoc are typically named <code>run-make/rustdoc-*/</code>.</p> |
| <h3 id="pretty-printer-tests"><a class="header" href="#pretty-printer-tests">Pretty-printer tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/pretty"><code>tests/pretty</code></a> exercise the "pretty-printing" functionality of |
| <code>rustc</code>. The <code>-Z unpretty</code> CLI option for <code>rustc</code> causes it to translate the |
| input source into various different formats, such as the Rust source after macro |
| expansion.</p> |
| <p>The pretty-printer tests have several <a href="tests/directives.html">directives</a> described below. |
| These commands can significantly change the behavior of the test, but the |
| default behavior without any commands is to:</p> |
| <ol> |
| <li>Run <code>rustc -Zunpretty=normal</code> on the source file.</li> |
| <li>Run <code>rustc -Zunpretty=normal</code> on the output of the previous step.</li> |
| <li>The output of the previous two steps should be the same.</li> |
| <li>Run <code>rustc -Zno-codegen</code> on the output to make sure that it can type check |
| (similar to <code>cargo check</code>).</li> |
| </ol> |
| <p>If any of the commands above fail, then the test fails.</p> |
| <p>The directives for pretty-printing tests are:</p> |
| <ul> |
| <li><code>pretty-mode</code> specifies the mode pretty-print tests should run in (that is, |
| the argument to <code>-Zunpretty</code>). The default is <code>normal</code> if not specified.</li> |
| <li><code>pretty-compare-only</code> causes a pretty test to only compare the pretty-printed |
| output (stopping after step 3 from above). It will not try to compile the |
| expanded output to type check it. This is needed for a pretty-mode that does |
| not expand to valid Rust, or for other situations where the expanded output |
| cannot be compiled.</li> |
| <li><code>pp-exact</code> is used to ensure a pretty-print test results in specific output. |
| If specified without a value, then it means the pretty-print output should |
| match the original source. If specified with a value, as in <code>//@ pp-exact:foo.pp</code>, it will ensure that the pretty-printed output matches the |
| contents of the given file. Otherwise, if <code>pp-exact</code> is not specified, then |
| the pretty-printed output will be pretty-printed one more time, and the output |
| of the two pretty-printing rounds will be compared to ensure that the |
| pretty-printed output converges to a steady state.</li> |
| </ul> |
| <h3 id="incremental-tests"><a class="header" href="#incremental-tests">Incremental tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/incremental"><code>tests/incremental</code></a> exercise incremental compilation. They use |
| <a href="tests/compiletest.html#revisions"><code>revisions</code> directive</a> to tell compiletest to run the compiler in a |
| series of steps.</p> |
| <p>Compiletest starts with an empty directory with the <code>-C incremental</code> flag, and |
| then runs the compiler for each revision, reusing the incremental results from |
| previous steps.</p> |
| <p>The revisions should start with:</p> |
| <ul> |
| <li><code>rpass</code> — the test should compile and run successfully</li> |
| <li><code>rfail</code> — the test should compile successfully, but the executable should fail to run</li> |
| <li><code>cfail</code> — the test should fail to compile</li> |
| </ul> |
| <p>To make the revisions unique, you should add a suffix like <code>rpass1</code> and |
| <code>rpass2</code>.</p> |
| <p>To simulate changing the source, compiletest also passes a <code>--cfg</code> flag with the |
| current revision name.</p> |
| <p>For example, this will run twice, simulating changing a function:</p> |
| <pre><code class="language-rust ignore">//@ revisions: rpass1 rpass2 |
| |
| #[cfg(rpass1)] |
| fn foo() { |
| println!("one"); |
| } |
| |
| #[cfg(rpass2)] |
| fn foo() { |
| println!("two"); |
| } |
| |
| fn main() { foo(); }</code></pre> |
| <p><code>cfail</code> tests support the <code>forbid-output</code> directive to specify that a certain |
| substring must not appear anywhere in the compiler output. This can be useful to |
| ensure certain errors do not appear, but this can be fragile as error messages |
| change over time, and a test may no longer be checking the right thing but will |
| still pass.</p> |
| <p><code>cfail</code> tests support the <code>should-ice</code> directive to specify that a test should |
| cause an Internal Compiler Error (ICE). This is a highly specialized directive |
| to check that the incremental cache continues to work after an ICE.</p> |
| <h3 id="debuginfo-tests"><a class="header" href="#debuginfo-tests">Debuginfo tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/debuginfo"><code>tests/debuginfo</code></a> test debuginfo generation. They build a |
| program, launch a debugger, and issue commands to the debugger. A single test |
| can work with cdb, gdb, and lldb.</p> |
| <p>Most tests should have the <code>//@ compile-flags: -g</code> directive or something |
| similar to generate the appropriate debuginfo.</p> |
| <p>To set a breakpoint on a line, add a <code>// #break</code> comment on the line.</p> |
| <p>The debuginfo tests consist of a series of debugger commands along with |
| "check" lines which specify output that is expected from the debugger.</p> |
| <p>The commands are comments of the form <code>// $DEBUGGER-command:$COMMAND</code> where |
| <code>$DEBUGGER</code> is the debugger being used and <code>$COMMAND</code> is the debugger command |
| to execute.</p> |
| <p>The debugger values can be:</p> |
| <ul> |
| <li><code>cdb</code></li> |
| <li><code>gdb</code></li> |
| <li><code>gdbg</code> — GDB without Rust support (versions older than 7.11)</li> |
| <li><code>gdbr</code> — GDB with Rust support</li> |
| <li><code>lldb</code></li> |
| <li><code>lldbg</code> — LLDB without Rust support</li> |
| <li><code>lldbr</code> — LLDB with Rust support (this no longer exists)</li> |
| </ul> |
| <p>The command to check the output are of the form <code>// $DEBUGGER-check:$OUTPUT</code> |
| where <code>$OUTPUT</code> is the output to expect.</p> |
| <p>For example, the following will build the test, start the debugger, set a |
| breakpoint, launch the program, inspect a value, and check what the debugger |
| prints:</p> |
| <pre><code class="language-rust ignore">//@ compile-flags: -g |
| |
| //@ lldb-command: run |
| //@ lldb-command: print foo |
| //@ lldb-check: $0 = 123 |
| |
| fn main() { |
| let foo = 123; |
| b(); // #break |
| } |
| |
| fn b() {}</code></pre> |
| <p>The following <a href="tests/directives.html">directives</a> are available to disable a test based on |
| the debugger currently being used:</p> |
| <ul> |
| <li><code>min-cdb-version: 10.0.18317.1001</code> — ignores the test if the version of cdb |
| is below the given version</li> |
| <li><code>min-gdb-version: 8.2</code> — ignores the test if the version of gdb is below the |
| given version</li> |
| <li><code>ignore-gdb-version: 9.2</code> — ignores the test if the version of gdb is equal |
| to the given version</li> |
| <li><code>ignore-gdb-version: 7.11.90 - 8.0.9</code> — ignores the test if the version of |
| gdb is in a range (inclusive)</li> |
| <li><code>min-lldb-version: 310</code> — ignores the test if the version of lldb is below |
| the given version</li> |
| <li><code>rust-lldb</code> — ignores the test if lldb is not contain the Rust plugin. NOTE: |
| The "Rust" version of LLDB doesn't exist anymore, so this will always be |
| ignored. This should probably be removed.</li> |
| </ul> |
| <p>By passing the <code>--debugger</code> option to compiletest, you can specify a single debugger to run tests with. |
| For example, <code>./x test tests/debuginfo -- --debugger gdb</code> will only test GDB commands.</p> |
| <blockquote> |
| <p><strong>Note on running lldb debuginfo tests locally</strong></p> |
| <p>If you want to run lldb debuginfo tests locally, then currently on Windows it |
| is required that:</p> |
| <ul> |
| <li>You have Python 3.10 installed.</li> |
| <li>You have <code>python310.dll</code> available in your <code>PATH</code> env var. This is not |
| provided by the standard Python installer you obtain from <code>python.org</code>; you |
| need to add this to <code>PATH</code> manually.</li> |
| </ul> |
| <p>Otherwise the lldb debuginfo tests can produce crashes in mysterious ways.</p> |
| </blockquote> |
| <blockquote> |
| <p><strong>Note on acquiring <code>cdb.exe</code> on Windows 11</strong></p> |
| <p><code>cdb.exe</code> is acquired alongside a suitable "Windows 11 SDK" which is part of |
| the "Desktop Development with C++" workload profile in a Visual Studio |
| installer (e.g. Visual Studio 2022 installer).</p> |
| <p><strong>HOWEVER</strong> this is not sufficient by default alone. If you need <code>cdb.exe</code>, |
| you must go to Installed Apps, find the newest "Windows Software Development |
| Kit" (and yes, this can still say <code>Windows 10.0.22161.3233</code> even though the OS |
| is called Windows 11). You must then click "Modify" -> "Change" and then |
| selected "Debugging Tools for Windows" in order to acquire <code>cdb.exe</code>.</p> |
| </blockquote> |
| <h3 id="codegen-tests"><a class="header" href="#codegen-tests">Codegen tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/codegen-llvm"><code>tests/codegen-llvm</code></a> test LLVM code generation. They compile the test |
| with the <code>--emit=llvm-ir</code> flag to emit LLVM IR. They then run the LLVM |
| <a href="https://llvm.org/docs/CommandGuide/FileCheck.html">FileCheck</a> tool. The test is annotated with various <code>// CHECK</code> comments to |
| check the generated code. See the <a href="https://llvm.org/docs/CommandGuide/FileCheck.html">FileCheck</a> documentation for a tutorial and |
| more information.</p> |
| <p>See also the <a href="tests/compiletest.html#assembly-tests">assembly tests</a> for a similar set of tests.</p> |
| <p>If you need to work with <code>#![no_std]</code> cross-compiling tests, consult the |
| <a href="tests/./minicore.html"><code>minicore</code> test auxiliary</a> chapter.</p> |
| <h3 id="assembly-tests"><a class="header" href="#assembly-tests">Assembly tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/assembly-llvm"><code>tests/assembly-llvm</code></a> test LLVM assembly output. They compile the test |
| with the <code>--emit=asm</code> flag to emit a <code>.s</code> file with the assembly output. They |
| then run the LLVM <a href="https://llvm.org/docs/CommandGuide/FileCheck.html">FileCheck</a> tool.</p> |
| <p>Each test should be annotated with the <code>//@ assembly-output:</code> directive with a |
| value of either <code>emit-asm</code> or <code>ptx-linker</code> to indicate the type of assembly |
| output.</p> |
| <p>Then, they should be annotated with various <code>// CHECK</code> comments to check the |
| assembly output. See the <a href="https://llvm.org/docs/CommandGuide/FileCheck.html">FileCheck</a> documentation for a tutorial and more |
| information.</p> |
| <p>See also the <a href="tests/compiletest.html#codegen-tests">codegen tests</a> for a similar set of tests.</p> |
| <p>If you need to work with <code>#![no_std]</code> cross-compiling tests, consult the |
| <a href="tests/./minicore.html"><code>minicore</code> test auxiliary</a> chapter.</p> |
| <h3 id="codegen-units-tests"><a class="header" href="#codegen-units-tests">Codegen-units tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/codegen-units"><code>tests/codegen-units</code></a> test the |
| <a href="tests/../backend/monomorph.html">monomorphization</a> collector and CGU partitioning.</p> |
| <p>These tests work by running <code>rustc</code> with a flag to print the result of the |
| monomorphization collection pass, i.e., <code>-Zprint-mono-items</code>, and then special |
| annotations in the file are used to compare against that.</p> |
| <p>Then, the test should be annotated with comments of the form <code>//~ MONO_ITEM name</code> where <code>name</code> is the monomorphized string printed by rustc like <code>fn <u32 as Trait>::foo</code>.</p> |
| <p>To check for CGU partitioning, a comment of the form <code>//~ MONO_ITEM name @@ cgu</code> |
| where <code>cgu</code> is a space separated list of the CGU names and the linkage |
| information in brackets. For example: <code>//~ MONO_ITEM static function::FOO @@ statics[Internal]</code></p> |
| <h3 id="mir-opt-tests"><a class="header" href="#mir-opt-tests">Mir-opt tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/mir-opt"><code>tests/mir-opt</code></a> check parts of the generated MIR to make sure it |
| is generated correctly and is doing the expected optimizations. Check out the |
| <a href="tests/../mir/optimizations.html">MIR Optimizations</a> chapter for more.</p> |
| <p>Compiletest will build the test with several flags to dump the MIR output and |
| set a baseline for optimizations:</p> |
| <ul> |
| <li><code>-Copt-level=1</code></li> |
| <li><code>-Zdump-mir=all</code></li> |
| <li><code>-Zmir-opt-level=4</code></li> |
| <li><code>-Zvalidate-mir</code></li> |
| <li><code>-Zdump-mir-exclude-pass-number</code></li> |
| </ul> |
| <p>The test should be annotated with <code>// EMIT_MIR</code> comments that specify files that |
| will contain the expected MIR output. You can use <code>x test --bless</code> to create the |
| initial expected files.</p> |
| <p>There are several forms the <code>EMIT_MIR</code> comment can take:</p> |
| <ul> |
| <li> |
| <p><code>// EMIT_MIR $MIR_PATH.mir</code> — This will check that the given filename matches |
| the exact output from the MIR dump. For example, |
| <code>my_test.main.SimplifyCfg-elaborate-drops.after.mir</code> will load that file from |
| the test directory, and compare it against the dump from rustc.</p> |
| <p>Checking the "after" file (which is after optimization) is useful if you are |
| interested in the final state after an optimization. Some rare cases may want |
| to use the "before" file for completeness.</p> |
| </li> |
| <li> |
| <p><code>// EMIT_MIR $MIR_PATH.diff</code> — where <code>$MIR_PATH</code> is the filename of the MIR |
| dump, such as <code>my_test_name.my_function.EarlyOtherwiseBranch</code>. Compiletest |
| will diff the <code>.before.mir</code> and <code>.after.mir</code> files, and compare the diff |
| output to the expected <code>.diff</code> file from the <code>EMIT_MIR</code> comment.</p> |
| <p>This is useful if you want to see how an optimization changes the MIR.</p> |
| </li> |
| <li> |
| <p><code>// EMIT_MIR $MIR_PATH.dot</code> — When using specific flags that dump additional |
| MIR data (e.g. <code>-Z dump-mir-graphviz</code> to produce <code>.dot</code> files), this will |
| check that the output matches the given file.</p> |
| </li> |
| </ul> |
| <p>By default 32 bit and 64 bit targets use the same dump files, which can be |
| problematic in the presence of pointers in constants or other bit width |
| dependent things. In that case you can add <code>// EMIT_MIR_FOR_EACH_BIT_WIDTH</code> to |
| your test, causing separate files to be generated for 32bit and 64bit systems.</p> |
| <h3 id="run-make-tests"><a class="header" href="#run-make-tests"><code>run-make</code> tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/run-make"><code>tests/run-make</code></a> and <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/run-make-cargo"><code>tests/run-make-cargo</code></a> are general-purpose |
| tests using Rust <em>recipes</em>, which are small programs (<code>rmake.rs</code>) allowing |
| arbitrary Rust code such as <code>rustc</code> invocations, and is supported by a |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/run-make-support"><code>run_make_support</code></a> library. Using Rust recipes provide the ultimate in |
| flexibility.</p> |
| <p><code>run-make</code> tests should be used if no other test suites better suit your needs.</p> |
| <p>The <code>run-make-cargo</code> test suite additionally builds an in-tree <code>cargo</code> to support |
| use cases that require testing in-tree <code>cargo</code> in conjunction with in-tree <code>rustc</code>. |
| The <code>run-make</code> test suite does not have access to in-tree <code>cargo</code> (so it can be the |
| faster-to-iterate test suite).</p> |
| <h4 id="using-rust-recipes"><a class="header" href="#using-rust-recipes">Using Rust recipes</a></h4> |
| <p>Each test should be in a separate directory with a <code>rmake.rs</code> Rust program, |
| called the <em>recipe</em>. A recipe will be compiled and executed by compiletest with |
| the <code>run_make_support</code> library linked in.</p> |
| <p>If you need new utilities or functionality, consider extending and improving the |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/run-make-support"><code>run_make_support</code></a> library.</p> |
| <p>Compiletest directives like <code>//@ only-<target></code> or <code>//@ ignore-<target></code> are |
| supported in <code>rmake.rs</code>, like in UI tests. However, revisions or building |
| auxiliary via directives are not currently supported.</p> |
| <p><code>rmake.rs</code> and <code>run-make-support</code> may <em>not</em> use any nightly/unstable features, |
| as they must be compilable by a stage 0 rustc that may be a beta or even stable |
| rustc.</p> |
| <h4 id="quickly-check-if-rmakers-tests-can-be-compiled"><a class="header" href="#quickly-check-if-rmakers-tests-can-be-compiled">Quickly check if <code>rmake.rs</code> tests can be compiled</a></h4> |
| <p>You can quickly check if <code>rmake.rs</code> tests can be compiled without having to |
| build stage1 rustc by forcing <code>rmake.rs</code> to be compiled with the stage0 |
| compiler:</p> |
| <pre><code class="language-bash">$ COMPILETEST_FORCE_STAGE0=1 x test --stage 0 tests/run-make/<test-name> |
| </code></pre> |
| <p>Of course, some tests will not successfully <em>run</em> in this way.</p> |
| <h4 id="using-rust-analyzer-with-rmakers"><a class="header" href="#using-rust-analyzer-with-rmakers">Using rust-analyzer with <code>rmake.rs</code></a></h4> |
| <p>Like other test programs, the <code>rmake.rs</code> scripts used by run-make tests do not |
| have rust-analyzer integration by default.</p> |
| <p>To work around this when working on a particular test, temporarily create a |
| <code>Cargo.toml</code> file in the test's directory |
| (e.g. <code>tests/run-make/sysroot-crates-are-unstable/Cargo.toml</code>) |
| with these contents:</p> |
| <div class="warning"> |
| <p>Be careful not to add this <code>Cargo.toml</code> or its <code>Cargo.lock</code> to your actual PR!</p> |
| </div> |
| <pre><code class="language-toml"># Convince cargo that this isn't part of an enclosing workspace. |
| [workspace] |
| |
| [package] |
| name = "rmake" |
| version = "0.1.0" |
| edition = "2021" |
| |
| [dependencies] |
| run_make_support = { path = "../../../src/tools/run-make-support" } |
| |
| [[bin]] |
| name = "rmake" |
| path = "rmake.rs" |
| </code></pre> |
| <p>Then add a corresponding entry to <code>"rust-analyzer.linkedProjects"</code> |
| (e.g. in <code>.vscode/settings.json</code>):</p> |
| <pre><code class="language-json">"rust-analyzer.linkedProjects": [ |
| "tests/run-make/sysroot-crates-are-unstable/Cargo.toml" |
| ], |
| </code></pre> |
| <h3 id="coverage-tests"><a class="header" href="#coverage-tests">Coverage tests</a></h3> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/coverage"><code>tests/coverage</code></a> are shared by multiple test modes that test |
| coverage instrumentation in different ways. Running the <code>coverage</code> test suite |
| will automatically run each test in all of the different coverage modes.</p> |
| <p>Each mode also has an alias to run the coverage tests in just that mode:</p> |
| <pre><code class="language-bash">./x test coverage # runs all of tests/coverage in all coverage modes |
| ./x test tests/coverage # same as above |
| |
| ./x test tests/coverage/if.rs # runs the specified test in all coverage modes |
| |
| ./x test coverage-map # runs all of tests/coverage in "coverage-map" mode only |
| ./x test coverage-run # runs all of tests/coverage in "coverage-run" mode only |
| |
| ./x test coverage-map -- tests/coverage/if.rs # runs the specified test in "coverage-map" mode only |
| </code></pre> |
| <p>If a particular test should not be run in one of the coverage test modes for |
| some reason, use the <code>//@ ignore-coverage-map</code> or <code>//@ ignore-coverage-run</code> |
| directives.</p> |
| <h4 id="coverage-map-suite"><a class="header" href="#coverage-map-suite"><code>coverage-map</code> suite</a></h4> |
| <p>In <code>coverage-map</code> mode, these tests verify the mappings between source code |
| regions and coverage counters that are emitted by LLVM. They compile the test |
| with <code>--emit=llvm-ir</code>, then use a custom tool (<a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/coverage-dump"><code>src/tools/coverage-dump</code></a>) to |
| extract and pretty-print the coverage mappings embedded in the IR. These tests |
| don't require the profiler runtime, so they run in PR CI jobs and are easy to |
| run/bless locally.</p> |
| <p>These coverage map tests can be sensitive to changes in MIR lowering or MIR |
| optimizations, producing mappings that are different but produce identical |
| coverage reports.</p> |
| <p>As a rule of thumb, any PR that doesn't change coverage-specific code should |
| <strong>feel free to re-bless</strong> the <code>coverage-map</code> tests as necessary, without |
| worrying about the actual changes, as long as the <code>coverage-run</code> tests still |
| pass.</p> |
| <h4 id="coverage-run-suite"><a class="header" href="#coverage-run-suite"><code>coverage-run</code> suite</a></h4> |
| <p>In <code>coverage-run</code> mode, these tests perform an end-to-end test of coverage |
| reporting. They compile a test program with coverage instrumentation, run that |
| program to produce raw coverage data, and then use LLVM tools to process that |
| data into a human-readable code coverage report.</p> |
| <p>Instrumented binaries need to be linked against the LLVM profiler runtime, so |
| <code>coverage-run</code> tests are <strong>automatically skipped</strong> unless the profiler runtime |
| is enabled in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml"># bootstrap.toml |
| [build] |
| profiler = true |
| </code></pre> |
| <p>This also means that they typically don't run in PR CI jobs, though they do run |
| as part of the full set of CI jobs used for merging.</p> |
| <h4 id="coverage-run-rustdoc-suite"><a class="header" href="#coverage-run-rustdoc-suite"><code>coverage-run-rustdoc</code> suite</a></h4> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/coverage-run-rustdoc"><code>tests/coverage-run-rustdoc</code></a> also run instrumented doctests and |
| include them in the coverage report. This avoids having to build rustdoc when |
| only running the main <code>coverage</code> suite.</p> |
| <h3 id="crash-tests"><a class="header" href="#crash-tests">Crash tests</a></h3> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/tests/crashes"><code>tests/crashes</code></a> serve as a collection of tests that are expected to cause the |
| compiler to ICE, panic or crash in some other way, so that accidental fixes are |
| tracked. Formerly, this was done at <a href="https://github.com/rust-lang/glacier">https://github.com/rust-lang/glacier</a> but |
| doing it inside the rust-lang/rust testsuite is more convenient.</p> |
| <p>It is imperative that a test in the suite causes rustc to ICE, panic, or |
| crash in some other way. A test will "pass" if rustc exits with an exit status |
| other than 1 or 0.</p> |
| <p>If you want to see verbose stdout/stderr, you need to set |
| <code>COMPILETEST_VERBOSE_CRASHES=1</code>, e.g.</p> |
| <pre><code class="language-bash">$ COMPILETEST_VERBOSE_CRASHES=1 ./x test tests/crashes/999999.rs --stage 1 |
| </code></pre> |
| <p>Anyone can add <a href="https://github.com/rust-lang/rust/issues?q=is%3Aissue+state%3Aopen+label%3AI-ICE%2CI-crash+label%3AT-compiler+label%3AS-has-mcve+-label%3AS-bug-has-test">"untracked" crashes</a> from the issue tracker. It's strongly |
| recommended to include test cases from several issues in a single PR. |
| When you do so, each issue number should be noted in the file name (<code>12345.rs</code> |
| should suffice) and also inside the file by means of a <code>//@ known-bug: #12345</code> |
| directive. Please <a href="https://forge.rust-lang.org/release/issue-triaging.html#applying-and-removing-labels">label</a> the relevant issues with <code>S-bug-has-test</code> |
| once your PR is merged.</p> |
| <p>If you happen to fix one of the crashes, please move it to a fitting |
| subdirectory in <code>tests/ui</code> and give it a meaningful name. Please add a doc |
| comment at the top of the file explaining why this test exists, even better if |
| you can briefly explain how the example causes rustc to crash previously and |
| what was done to prevent rustc to ICE / panic / crash.</p> |
| <p>Adding</p> |
| <pre><code class="language-text">Fixes #NNNNN |
| Fixes #MMMMM |
| </code></pre> |
| <p>to the description of your pull request will ensure the corresponding tickets be closed |
| automatically upon merge.</p> |
| <p>Make sure that your fix actually fixes the root cause of the issue and not just |
| a subset first. The issue numbers can be found in the file name or the <code>//@ known-bug</code> directive inside the test file.</p> |
| <h2 id="building-auxiliary-crates"><a class="header" href="#building-auxiliary-crates">Building auxiliary crates</a></h2> |
| <p>It is common that some tests require additional auxiliary crates to be compiled. |
| There are multiple <a href="tests/directives.html">directives</a> to assist with that:</p> |
| <ul> |
| <li><code>aux-build</code></li> |
| <li><code>aux-crate</code></li> |
| <li><code>aux-bin</code></li> |
| <li><code>aux-codegen-backend</code></li> |
| <li><code>proc-macro</code></li> |
| </ul> |
| <p><code>aux-build</code> will build a separate crate from the named source file. The source |
| file should be in a directory called <code>auxiliary</code> beside the test file.</p> |
| <pre><code class="language-rust ignore">//@ aux-build: my-helper.rs |
| |
| extern crate my_helper; |
| // ... You can use my_helper.</code></pre> |
| <p>The aux crate will be built as a dylib if possible (unless on a platform that |
| does not support them, or the <code>no-prefer-dynamic</code> header is specified in the aux |
| file). The <code>-L</code> flag is used to find the extern crates.</p> |
| <p><code>aux-crate</code> is very similar to <code>aux-build</code>. However, it uses the <code>--extern</code> flag |
| to link to the extern crate to make the crate be available as an extern prelude. |
| That allows you to specify the additional syntax of the <code>--extern</code> flag, such as |
| renaming a dependency. For example, <code>//@ aux-crate:foo=bar.rs</code> will compile |
| <code>auxiliary/bar.rs</code> and make it available under then name <code>foo</code> within the test. |
| This is similar to how Cargo does dependency renaming.</p> |
| <p><code>aux-bin</code> is similar to <code>aux-build</code> but will build a binary instead of a |
| library. The binary will be available in <code>auxiliary/bin</code> relative to the working |
| directory of the test.</p> |
| <p><code>aux-codegen-backend</code> is similar to <code>aux-build</code>, but will then pass the compiled |
| dylib to <code>-Zcodegen-backend</code> when building the main file. This will only work |
| for tests in <code>tests/ui-fulldeps</code>, since it requires the use of compiler crates.</p> |
| <h3 id="auxiliary-proc-macro"><a class="header" href="#auxiliary-proc-macro">Auxiliary proc-macro</a></h3> |
| <p>If you want a proc-macro dependency, then you can use the <code>proc-macro</code> |
| directive. This directive behaves just like <code>aux-build</code>, i.e. that you should |
| place the proc-macro test auxiliary file under a <code>auxiliary</code> folder under the |
| same parent folder as the main test file. However, it also has four additional |
| preset behavior compared to <code>aux-build</code> for the proc-macro test auxiliary:</p> |
| <ol> |
| <li>The aux test file is built with <code>--crate-type=proc-macro</code>.</li> |
| <li>The aux test file is built without <code>-C prefer-dynamic</code>, i.e. it will not try |
| to produce a dylib for the aux crate.</li> |
| <li>The aux crate is made available to the test file via extern prelude with |
| <code>--extern <aux_crate_name></code>. Note that since UI tests default to edition |
| 2015, you still need to specify <code>extern <aux_crate_name></code> unless the main |
| test file is using an edition that is 2018 or newer if you want to use the |
| aux crate name in a <code>use</code> import.</li> |
| <li>The <code>proc_macro</code> crate is made available as an extern prelude module. Same |
| edition 2015 vs newer edition distinction for <code>extern proc_macro;</code> applies.</li> |
| </ol> |
| <p>For example, you might have a test <code>tests/ui/cat/meow.rs</code> and proc-macro |
| auxiliary <code>tests/ui/cat/auxiliary/whiskers.rs</code>:</p> |
| <pre><code class="language-text">tests/ui/cat/ |
| meow.rs # main test file |
| auxiliary/whiskers.rs # auxiliary |
| </code></pre> |
| <pre><code class="language-rs">// tests/ui/cat/meow.rs |
| |
| //@ proc-macro: whiskers.rs |
| |
| extern crate whiskers; // needed as ui test defaults to edition 2015 |
| |
| fn main() { |
| whiskers::identity!(); |
| } |
| </code></pre> |
| <pre><code class="language-rs">// tests/ui/cat/auxiliary/whiskers.rs |
| |
| extern crate proc_macro; |
| use proc_macro::*; |
| |
| #[proc_macro] |
| pub fn identity(ts: TokenStream) -> TokenStream { |
| ts |
| } |
| </code></pre> |
| <blockquote> |
| <p><strong>Note</strong>: The <code>proc-macro</code> header currently does not work with the |
| <code>build-aux-doc</code> header for rustdoc tests. In that case, you will need to use |
| the <code>aux-build</code> header, and use <code>#![crate_type="proc_macro"]</code>, and <code>//@ force-host</code> and <code>//@ no-prefer-dynamic</code> headers in the proc-macro.</p> |
| </blockquote> |
| <h2 id="revisions"><a class="header" href="#revisions">Revisions</a></h2> |
| <p>Revisions allow a single test file to be used for multiple tests. This is done |
| by adding a special directive at the top of the file:</p> |
| <pre><code class="language-rust ignore">//@ revisions: foo bar baz</code></pre> |
| <p>This will result in the test being compiled (and tested) three times, once with |
| <code>--cfg foo</code>, once with <code>--cfg bar</code>, and once with <code>--cfg baz</code>. You can therefore |
| use <code>#[cfg(foo)]</code> etc within the test to tweak each of these results.</p> |
| <p>You can also customize directives and expected error messages to a particular |
| revision. To do this, add <code>[revision-name]</code> after the <code>//@</code> for directives, and |
| after <code>//</code> for UI error annotations, like so:</p> |
| <pre><code class="language-rust ignore">// A flag to pass in only for cfg `foo`: |
| //@[foo]compile-flags: -Z verbose-internals |
| |
| #[cfg(foo)] |
| fn test_foo() { |
| let x: usize = 32_u32; //[foo]~ ERROR mismatched types |
| }</code></pre> |
| <p>Multiple revisions can be specified in a comma-separated list, such as |
| <code>//[foo,bar,baz]~^</code>.</p> |
| <p>In test suites that use the LLVM <a href="https://llvm.org/docs/CommandGuide/FileCheck.html">FileCheck</a> tool, the current revision name is |
| also registered as an additional prefix for FileCheck directives:</p> |
| <pre><code class="language-rust ignore">//@ revisions: NORMAL COVERAGE |
| //@[COVERAGE] compile-flags: -Cinstrument-coverage |
| //@[COVERAGE] needs-profiler-runtime |
| |
| // COVERAGE: @__llvm_coverage_mapping |
| // NORMAL-NOT: @__llvm_coverage_mapping |
| |
| // CHECK: main |
| fn main() {}</code></pre> |
| <p>Note that not all directives have meaning when customized to a revision. For |
| example, the <code>ignore-test</code> directives (and all "ignore" directives) currently |
| only apply to the test as a whole, not to particular revisions. The only |
| directives that are intended to really work when customized to a revision are |
| error patterns and compiler flags.</p> |
| <!-- date-check jul 2023 --> |
| <p>The following test suites support revisions:</p> |
| <ul> |
| <li>ui</li> |
| <li>assembly</li> |
| <li>codegen</li> |
| <li>coverage</li> |
| <li>debuginfo</li> |
| <li>rustdoc UI tests</li> |
| <li>incremental (these are special in that they inherently cannot be run in |
| parallel)</li> |
| </ul> |
| <h3 id="ignoring-unused-revision-names"><a class="header" href="#ignoring-unused-revision-names">Ignoring unused revision names</a></h3> |
| <p>Normally, revision names mentioned in other directives and error annotations |
| must correspond to an actual revision declared in a <code>revisions</code> directive. This is |
| enforced by an <code>./x test tidy</code> check.</p> |
| <p>If a revision name needs to be temporarily removed from the revision list for |
| some reason, the above check can be suppressed by adding the revision name to an |
| <code>//@ unused-revision-names:</code> header instead.</p> |
| <p>Specifying an unused name of <code>*</code> (i.e. <code>//@ unused-revision-names: *</code>) will |
| permit any unused revision name to be mentioned.</p> |
| <h2 id="compare-modes"><a class="header" href="#compare-modes">Compare modes</a></h2> |
| <p>Compiletest can be run in different modes, called <em>compare modes</em>, which can be |
| used to compare the behavior of all tests with different compiler flags enabled. |
| This can help highlight what differences might appear with certain flags, and |
| check for any problems that might arise.</p> |
| <p>To run the tests in a different mode, you need to pass the <code>--compare-mode</code> CLI |
| flag:</p> |
| <pre><code class="language-bash">./x test tests/ui --compare-mode=chalk |
| </code></pre> |
| <p>The possible compare modes are:</p> |
| <ul> |
| <li><code>polonius</code> — Runs with Polonius with <code>-Zpolonius</code>.</li> |
| <li><code>chalk</code> — Runs with Chalk with <code>-Zchalk</code>.</li> |
| <li><code>split-dwarf</code> — Runs with unpacked split-DWARF with |
| <code>-Csplit-debuginfo=unpacked</code>.</li> |
| <li><code>split-dwarf-single</code> — Runs with packed split-DWARF with |
| <code>-Csplit-debuginfo=packed</code>.</li> |
| </ul> |
| <p>See <a href="tests/ui.html#compare-modes">UI compare modes</a> for more information about how UI |
| tests support different output for different modes.</p> |
| <p>In CI, compare modes are only used in one Linux builder, and only with the |
| following settings:</p> |
| <ul> |
| <li><code>tests/debuginfo</code>: Uses <code>split-dwarf</code> mode. This helps ensure that none of the |
| debuginfo tests are affected when enabling split-DWARF.</li> |
| </ul> |
| <p>Note that compare modes are separate to <a href="tests/compiletest.html#revisions">revisions</a>. All revisions |
| are tested when running <code>./x test tests/ui</code>, however compare-modes must be |
| manually run individually via the <code>--compare-mode</code> flag.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="ui-tests"><a class="header" href="#ui-tests">UI tests</a></h1> |
| <p>UI tests are a particular <a href="tests/compiletest.html#test-suites">test suite</a> of |
| compiletest.</p> |
| <h2 id="introduction-1"><a class="header" href="#introduction-1">Introduction</a></h2> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui"><code>tests/ui</code></a> are a collection of general-purpose tests which |
| primarily focus on validating the console output of the compiler, but can be |
| used for many other purposes. For example, tests can also be configured to <a href="tests/ui.html#controlling-passfail-expectations">run |
| the resulting program</a> to verify its |
| behavior.</p> |
| <p>For a survey of each subdirectory's purpose under <code>tests/ui</code>, consult the |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/ui/README.md">README.md</a>. |
| This is useful if you write a new test, and are looking for a category to |
| place it in.</p> |
| <p>If you need to work with <code>#![no_std]</code> cross-compiling tests, consult the |
| <a href="tests/./minicore.html"><code>minicore</code> test auxiliary</a> chapter.</p> |
| <h2 id="general-structure-of-a-test"><a class="header" href="#general-structure-of-a-test">General structure of a test</a></h2> |
| <p>A test consists of a Rust source file located in the <code>tests/ui</code> directory. |
| <strong>Tests must be placed in the appropriate subdirectory</strong> based on their purpose |
| and testing category - placing tests directly in <code>tests/ui</code> is not permitted.</p> |
| <p>Compiletest will use <code>rustc</code> to compile the test, and compare the output against |
| the expected output which is stored in a <code>.stdout</code> or <code>.stderr</code> file located |
| next to the test. See <a href="tests/ui.html#output-comparison">Output comparison</a> for more.</p> |
| <p>Additionally, errors and warnings should be annotated with comments within the |
| source file. See <a href="tests/ui.html#error-annotations">Error annotations</a> for more.</p> |
| <p>Compiletest <a href="tests/directives.html">directives</a> in the form of special comments prefixed |
| with <code>//@</code> control how the test is compiled and what the expected behavior is.</p> |
| <p>Tests are expected to fail to compile, since most tests are testing compiler |
| errors. You can change that behavior with a directive, see <a href="tests/ui.html#controlling-passfail-expectations">Controlling |
| pass/fail expectations</a>.</p> |
| <p>By default, a test is built as an executable binary. If you need a different |
| crate type, you can use the <code>#![crate_type]</code> attribute to set it as needed.</p> |
| <h2 id="output-comparison"><a class="header" href="#output-comparison">Output comparison</a></h2> |
| <p>UI tests store the expected output from the compiler in <code>.stderr</code> and <code>.stdout</code> |
| snapshots next to the test. You normally generate these files with the <code>--bless</code> |
| CLI option, and then inspect them manually to verify they contain what you |
| expect.</p> |
| <p>The output is normalized to ignore unwanted differences, see the |
| <a href="tests/ui.html#normalization">Normalization</a> section. If the file is missing, then |
| compiletest expects the corresponding output to be empty.</p> |
| <p>A common reason to use normalization, revisions, and most of the other following tools, |
| is to account for platform differences. Consider alternatives to these tools, like |
| e.g. using the <code>extern "rust-invalid"</code> ABI that is invalid on every platform |
| instead of fixing the test to use cross-compilation and testing every possibly-invalid ABI.</p> |
| <p>There can be multiple stdout/stderr files. The general form is:</p> |
| <pre><code class="language-text">*test-name*`.`*revision*`.`*compare_mode*`.`*extension* |
| </code></pre> |
| <ul> |
| <li><em>test-name</em> cannot contain dots. This is so that the general form of test |
| output filenames have a predictable form we can pattern match on in order to |
| track stray test output files.</li> |
| <li><em>revision</em> is the <a href="tests/ui.html#cfg-revisions">revision</a> name. This is not included when |
| not using revisions.</li> |
| <li><em>compare_mode</em> is the <a href="tests/ui.html#compare-modes">compare mode</a>. This will only be |
| checked when the given compare mode is active. If the file does not exist, |
| then compiletest will check for a file without the compare mode.</li> |
| <li><em>extension</em> is the kind of output being checked: |
| <ul> |
| <li><code>stderr</code> — compiler stderr</li> |
| <li><code>stdout</code> — compiler stdout</li> |
| <li><code>run.stderr</code> — stderr when running the test</li> |
| <li><code>run.stdout</code> — stdout when running the test</li> |
| <li><code>64bit.stderr</code> — compiler stderr with <code>stderr-per-bitwidth</code> directive on a |
| 64-bit target</li> |
| <li><code>32bit.stderr</code> — compiler stderr with <code>stderr-per-bitwidth</code> directive on a |
| 32-bit target</li> |
| </ul> |
| </li> |
| </ul> |
| <p>A simple example would be <code>foo.stderr</code> next to a <code>foo.rs</code> test. |
| A more complex example would be <code>foo.my-revision.polonius.stderr</code>.</p> |
| <p>There are several <a href="tests/directives.html">directives</a> which will change how compiletest |
| will check for output files:</p> |
| <ul> |
| <li><code>stderr-per-bitwidth</code> — checks separate output files based on the target |
| pointer width. Consider using the <code>normalize-stderr</code> directive instead (see |
| <a href="tests/ui.html#normalization">Normalization</a>).</li> |
| <li><code>dont-check-compiler-stderr</code> — Ignores stderr from the compiler.</li> |
| <li><code>dont-check-compiler-stdout</code> — Ignores stdout from the compiler.</li> |
| <li><code>compare-output-by-lines</code> — Some tests have non-deterministic orders of output, so we need to compare by lines.</li> |
| </ul> |
| <p>UI tests run with <code>-Zdeduplicate-diagnostics=no</code> flag which disables rustc's |
| built-in diagnostic deduplication mechanism. This means you may see some |
| duplicate messages in the output. This helps illuminate situations where |
| duplicate diagnostics are being generated.</p> |
| <h3 id="normalization"><a class="header" href="#normalization">Normalization</a></h3> |
| <p>The compiler output is normalized to eliminate output difference between |
| platforms, mainly about filenames.</p> |
| <p>Compiletest makes the following replacements on the compiler output:</p> |
| <ul> |
| <li>The directory where the test is defined is replaced with <code>$DIR</code>. Example: |
| <code>/path/to/rust/tests/ui/error-codes</code></li> |
| <li>The directory to the standard library source is replaced with <code>$SRC_DIR</code>. |
| Example: <code>/path/to/rust/library</code></li> |
| <li>Line and column numbers for paths in <code>$SRC_DIR</code> are replaced with <code>LL:COL</code>. |
| This helps ensure that changes to the layout of the standard library do not |
| cause widespread changes to the <code>.stderr</code> files. Example: |
| <code>$SRC_DIR/alloc/src/sync.rs:53:46</code></li> |
| <li>The base directory where the test's output goes is replaced with |
| <code>$TEST_BUILD_DIR</code>. This only comes up in a few rare circumstances. Example: |
| <code>/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui</code></li> |
| <li>The real directory to the standard library source is replaced with <code>$SRC_DIR_REAL</code>.</li> |
| <li>The real directory to the compiler source is replaced with <code>$COMPILER_DIR_REAL</code>.</li> |
| <li>Tabs are replaced with <code>\t</code>.</li> |
| <li>Backslashes (<code>\</code>) are converted to forward slashes (<code>/</code>) within paths (using a |
| heuristic). This helps normalize differences with Windows-style paths.</li> |
| <li>CRLF newlines are converted to LF.</li> |
| <li>Error line annotations like <code>//~ ERROR some message</code> are removed.</li> |
| <li>Various v0 and legacy symbol hashes are replaced with placeholders like |
| <code>[HASH]</code> or <code><SYMBOL_HASH></code>.</li> |
| </ul> |
| <p>Additionally, the compiler is run with the <code>-Z ui-testing</code> flag which causes |
| the compiler itself to apply some changes to the diagnostic output to make it |
| more suitable for UI testing.</p> |
| <p>For example, it will anonymize line numbers in the output (line numbers |
| prefixing each source line are replaced with <code>LL</code>). In extremely rare |
| situations, this mode can be disabled with the directive <code>//@ compile-flags: -Z ui-testing=no</code>.</p> |
| <p>Note: The line and column numbers for <code>--></code> lines pointing to the test are <em>not</em> |
| normalized, and left as-is. This ensures that the compiler continues to point to |
| the correct location, and keeps the stderr files readable. Ideally all |
| line/column information would be retained, but small changes to the source |
| causes large diffs, and more frequent merge conflicts and test errors.</p> |
| <p>Sometimes these built-in normalizations are not enough. In such cases, you may |
| provide custom normalization rules using <code>normalize-*</code> directives, e.g.</p> |
| <pre><code class="language-rust ignore">//@ normalize-stdout: "foo" -> "bar" |
| //@ normalize-stderr: "foo" -> "bar" |
| //@ normalize-stderr-32bit: "fn\(\) \(32 bits\)" -> "fn\(\) \($$PTR bits\)" |
| //@ normalize-stderr-64bit: "fn\(\) \(64 bits\)" -> "fn\(\) \($$PTR bits\)"</code></pre> |
| <p>This tells the test, on 32-bit platforms, whenever the compiler writes <code>fn() (32 bits)</code> to stderr, it should be normalized to read <code>fn() ($PTR bits)</code> instead. |
| Similar for 64-bit. The replacement is performed by regexes using default regex |
| flavor provided by <code>regex</code> crate.</p> |
| <p>The corresponding reference file will use the normalized output to test both |
| 32-bit and 64-bit platforms:</p> |
| <pre><code class="language-text">... |
| | |
| = note: source type: fn() ($PTR bits) |
| = note: target type: u16 (16 bits) |
| ... |
| </code></pre> |
| <p>Please see <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui/transmute/main.rs"><code>ui/transmute/main.rs</code></a> and <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui/transmute/main.stderr"><code>main.stderr</code></a> for a concrete |
| usage example.</p> |
| <h2 id="error-annotations"><a class="header" href="#error-annotations">Error annotations</a></h2> |
| <p>Error annotations specify the errors that the compiler is expected to emit. They |
| are "attached" to the line in source where the error is located.</p> |
| <pre><code class="language-rust ignore">fn main() { |
| boom //~ ERROR cannot find value `boom` in this scope [E0425] |
| }</code></pre> |
| <p>Although UI tests have a <code>.stderr</code> file which contains the entire compiler |
| output, UI tests require that errors are also annotated within the source. This |
| redundancy helps avoid mistakes since the <code>.stderr</code> files are usually |
| auto-generated. It also helps to directly see where the error spans are expected |
| to point to by looking at one file instead of having to compare the <code>.stderr</code> |
| file with the source. Finally, they ensure that no additional unexpected errors |
| are generated.</p> |
| <p>They have several forms, but generally are a comment with the diagnostic level |
| (such as <code>ERROR</code>) and a substring of the expected error output. You don't have |
| to write out the entire message, just make sure to include the important part of |
| the message to make it self-documenting.</p> |
| <p>Most error annotations need to match with the line of the diagnostic. There are |
| several ways to match the message with the line (see the examples below):</p> |
| <ul> |
| <li><code>~</code>: Associates the error level and message with the <em>current</em> line</li> |
| <li><code>~^</code>: Associates the error level and message with the <em>previous</em> error |
| annotation line. Each caret (<code>^</code>) that you add adds a line to this, so <code>~^^^</code> |
| is three lines above the error annotation line.</li> |
| <li><code>~|</code>: Associates the error level and message with the <em>same</em> line as the |
| <em>previous comment</em>. This is more convenient than using multiple carets when |
| there are multiple messages associated with the same line.</li> |
| <li><code>~v</code>: Associates the error level and message with the <em>next</em> error |
| annotation line. Each symbol (<code>v</code>) that you add adds a line to this, so <code>~vvv</code> |
| is three lines below the error annotation line.</li> |
| </ul> |
| <p>Example:</p> |
| <pre><code class="language-rust ignore">let _ = same_line; //~ ERROR undeclared variable |
| fn meow(_: [u8]) {} |
| //~^ ERROR unsized |
| //~| ERROR anonymous parameters</code></pre> |
| <p>The space character between <code>//~</code> (or other variants) and the subsequent text is |
| negligible (i.e. there is no semantic difference between <code>//~ ERROR</code> and |
| <code>//~ERROR</code> although the former is more common in the codebase).</p> |
| <p><code>~? <diagnostic kind></code> (example being <code>~? ERROR</code>) |
| is used to match diagnostics <em>without</em> line info at all, |
| or where the line info is outside the main test file<sup class="footnote-reference" id="fr-main test file-1"><a href="#footnote-main test file">1</a></sup>. |
| These annotations can be placed on any line in the test file.</p> |
| <h3 id="error-annotation-examples"><a class="header" href="#error-annotation-examples">Error annotation examples</a></h3> |
| <p>Here are examples of error annotations on different lines of UI test source.</p> |
| <h4 id="positioned-on-error-line"><a class="header" href="#positioned-on-error-line">Positioned on error line</a></h4> |
| <p>Use the <code>//~ ERROR</code> idiom:</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let x = (1, 2, 3); |
| match x { |
| (_a, _x @ ..) => {} //~ ERROR `_x @` is not allowed in a tuple |
| _ => {} |
| } |
| }</code></pre> |
| <h4 id="positioned-below-error-line"><a class="header" href="#positioned-below-error-line">Positioned below error line</a></h4> |
| <p>Use the <code>//~^</code> idiom with number of carets in the string to indicate the number |
| of lines above. In the example below, the error line is four lines above the |
| error annotation line so four carets are included in the annotation.</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let x = (1, 2, 3); |
| match x { |
| (_a, _x @ ..) => {} // <- the error is on this line |
| _ => {} |
| } |
| } |
| //~^^^^ ERROR `_x @` is not allowed in a tuple</code></pre> |
| <h4 id="use-same-error-line-as-defined-on-error-annotation-line-above"><a class="header" href="#use-same-error-line-as-defined-on-error-annotation-line-above">Use same error line as defined on error annotation line above</a></h4> |
| <p>Use the <code>//~|</code> idiom to define the same error line as the error annotation |
| line above:</p> |
| <pre><code class="language-rust ignore">struct Binder(i32, i32, i32); |
| |
| fn main() { |
| let x = Binder(1, 2, 3); |
| match x { |
| Binder(_a, _x @ ..) => {} // <- the error is on this line |
| _ => {} |
| } |
| } |
| //~^^^^ ERROR `_x @` is not allowed in a tuple struct |
| //~| ERROR this pattern has 1 field, but the corresponding tuple struct has 3 fields [E0023]</code></pre> |
| <h4 id="positioned-above-error-line"><a class="header" href="#positioned-above-error-line">Positioned above error line</a></h4> |
| <p>Use the <code>//~v</code> idiom with number of v's in the string to indicate the number |
| of lines below. This is typically used in lexer or parser tests matching on errors like unclosed |
| delimiter or unclosed literal happening at the end of file.</p> |
| <pre><code class="language-rust ignore">// ignore-tidy-trailing-newlines |
| //~v ERROR this file contains an unclosed delimiter |
| fn main((ؼ</code></pre> |
| <h4 id="error-without-line-information"><a class="header" href="#error-without-line-information">Error without line information</a></h4> |
| <p>Use <code>//~?</code> to match an error without line information. |
| <code>//~?</code> is precise and will not match errors if their line information is available. |
| It should be preferred over <code>//@ error-pattern</code> |
| for tests wishing to match against compiler diagnostics, |
| due to <code>//@ error-pattern</code> being imprecise and non-exhaustive.</p> |
| <pre><code class="language-rust ignore">//@ compile-flags: --print yyyy |
| |
| //~? ERROR unknown print request: `yyyy`</code></pre> |
| <h3 id="error-pattern"><a class="header" href="#error-pattern"><code>error-pattern</code></a></h3> |
| <p>The <code>error-pattern</code> <a href="tests/directives.html">directive</a> can be used for runtime messages which don't |
| have a specific span, or, in exceptional cases, for compile time messages.</p> |
| <p>Let's think about this test:</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let a: *const [_] = &[1, 2, 3]; |
| unsafe { |
| let _b = (*a)[3]; |
| } |
| }</code></pre> |
| <p>We want to ensure this shows "index out of bounds", but we cannot use the <code>ERROR</code> |
| annotation since the runtime error doesn't have any span. Then it's time to use the |
| <code>error-pattern</code> directive:</p> |
| <pre><code class="language-rust ignore">//@ error-pattern: index out of bounds |
| fn main() { |
| let a: *const [_] = &[1, 2, 3]; |
| unsafe { |
| let _b = (*a)[3]; |
| } |
| }</code></pre> |
| <p>For strict testing of compile time output, try to use the line annotations <code>//~</code> as much as |
| possible, including <code>//~?</code> annotations for diagnostics without spans.</p> |
| <p>If the compile time output is target dependent or too verbose, use directive |
| <code>//@ dont-require-annotations: <diagnostic-kind></code> to make the line annotation checking |
| non-exhaustive. |
| Some of the compiler messages can stay uncovered by annotations in this mode.</p> |
| <p>For checking runtime output, <code>//@ check-run-results</code> may be preferable.</p> |
| <p>Only use <code>error-pattern</code> if none of the above works, such as when finding a |
| specific string pattern in a runtime panic output.</p> |
| <p>Line annotations <code>//~</code> and <code>error-pattern</code> are compatible and can be used in the same test.</p> |
| <h3 id="diagnostic-kinds-error-levels"><a class="header" href="#diagnostic-kinds-error-levels">Diagnostic kinds (error levels)</a></h3> |
| <p>The diagnostic kinds that you can have are:</p> |
| <ul> |
| <li><code>ERROR</code></li> |
| <li><code>WARN</code> (or <code>WARNING</code>)</li> |
| <li><code>NOTE</code></li> |
| <li><code>HELP</code></li> |
| <li><code>SUGGESTION</code></li> |
| <li><code>RAW</code></li> |
| </ul> |
| <p>The <code>SUGGESTION</code> kind is used for specifying what the expected replacement text |
| should be for a diagnostic suggestion. |
| The <code>RAW</code> kind can be used for matching on lines from non-structured output sometimes emitted |
| by the compiler instead of or in addition to structured json.</p> |
| <p><code>ERROR</code> and <code>WARN</code> kinds are required to be exhaustively covered by line annotations |
| <code>//~</code> by default.</p> |
| <p>Other kinds only need to be line-annotated if at least one annotation of that kind appears |
| in the test file. For example, one <code>//~ NOTE</code> will also require all other <code>//~ NOTE</code>s in the file |
| to be written out explicitly.</p> |
| <p>Use directive <code>//@ dont-require-annotations</code> to opt out of exhaustive annotations. |
| E.g. use <code>//@ dont-require-annotations: NOTE</code> to annotate notes selectively. |
| Avoid using this directive for <code>ERROR</code>s and <code>WARN</code>ings, unless there's a serious reason, like |
| target-dependent compiler output.</p> |
| <p>Some diagnostics are never required to be line-annotated, regardless of their kind or directives, |
| for example secondary lines of multiline diagnostics, |
| or ubiquitous diagnostics like <code>aborting due to N previous errors</code>.</p> |
| <p>UI tests use the <code>-A unused</code> flag by default to ignore all unused warnings, as |
| unused warnings are usually not the focus of a test. However, simple code |
| samples often have unused warnings. If the test is specifically testing an |
| unused warning, just add the appropriate <code>#![warn(unused)]</code> attribute as needed.</p> |
| <h3 id="cfg-revisions"><a class="header" href="#cfg-revisions"><code>cfg</code> revisions</a></h3> |
| <p>When using <a href="tests/compiletest.html#revisions">revisions</a>, different messages can be |
| conditionally checked based on the current revision. This is done by placing the |
| revision cfg name in brackets like this:</p> |
| <pre><code class="language-rust ignore">//@ edition:2018 |
| //@ revisions: mir thir |
| //@[thir] compile-flags: -Z thir-unsafeck |
| |
| async unsafe fn f() {} |
| |
| async fn g() { |
| f(); //~ ERROR call to unsafe function is unsafe |
| } |
| |
| fn main() { |
| f(); //[mir]~ ERROR call to unsafe function is unsafe |
| }</code></pre> |
| <p>In this example, the second error message is only emitted in the <code>mir</code> revision. |
| The <code>thir</code> revision only emits the first error.</p> |
| <p>If the <code>cfg</code> causes the compiler to emit different output, then a test can have |
| multiple <code>.stderr</code> files for the different outputs. In the example above, there |
| would be a <code>.mir.stderr</code> and <code>.thir.stderr</code> file with the different outputs of |
| the different revisions.</p> |
| <blockquote> |
| <p>Note: cfg revisions also work inside the source code with <code>#[cfg]</code> attributes.</p> |
| <p>By convention, the <code>FALSE</code> cfg is used to have an always-false config.</p> |
| </blockquote> |
| <h2 id="controlling-passfail-expectations"><a class="header" href="#controlling-passfail-expectations">Controlling pass/fail expectations</a></h2> |
| <p>By default, a UI test is expected to <strong>generate a compile error</strong> because most |
| of the tests are checking for invalid input and error diagnostics. However, you |
| can also make UI tests where compilation is expected to succeed, and you can |
| even run the resulting program. Just add one of the following |
| <a href="tests/directives.html">directives</a>:</p> |
| <ul> |
| <li>Pass directives: |
| <ul> |
| <li><code>//@ check-pass</code> — compilation should succeed but skip codegen |
| (which is expensive and isn't supposed to fail in most cases).</li> |
| <li><code>//@ build-pass</code> — compilation and linking should succeed but do |
| not run the resulting binary.</li> |
| <li><code>//@ run-pass</code> — compilation should succeed and running the resulting |
| binary should make it exit with code 0 which indicates success.</li> |
| </ul> |
| </li> |
| <li>Fail directives: |
| <ul> |
| <li><code>//@ check-fail</code> — compilation should fail (the codegen phase is skipped). |
| This is the default for UI tests.</li> |
| <li><code>//@ build-fail</code> — compilation should fail during the codegen phase. |
| This will run <code>rustc</code> twice: |
| <ul> |
| <li>First time is to ensure that the compile succeeds without the codegen phase</li> |
| <li>Second time is to ensure that the full compile fails</li> |
| </ul> |
| </li> |
| <li><code>//@ run-fail</code> — compilation should succeed, but running the resulting |
| binary should make it exit with a code in the range <code>1..=127</code> which |
| indicates regular failure. On targets without unwind support, crashes |
| are also accepted.</li> |
| <li><code>//@ run-crash</code> — compilation should succeed, but running the resulting |
| binary should fail with a crash. Crashing is defined as "not exiting with |
| a code in the range <code>0..=127</code>". Example on Linux: Termination by <code>SIGABRT</code> |
| or <code>SIGSEGV</code>. Example on Windows: Exiting with the code for |
| <code>STATUS_ILLEGAL_INSTRUCTION</code> (<code>0xC000001D</code>).</li> |
| <li><code>//@ run-fail-or-crash</code> — compilation should succeed, but running the |
| resulting binary should either <code>run-fail</code> or <code>run-crash</code>. Useful if a test |
| crashes on some targets but just fails on others.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>For <code>run-pass</code>. <code>run-fail</code>, <code>run-crash</code> and <code>run-fail-or-crash</code> tests, by |
| default the output of the program itself is not checked.</p> |
| <p>If you want to check the output of running the program, include the |
| <code>check-run-results</code> directive. This will check for a <code>.run.stderr</code> and |
| <code>.run.stdout</code> files to compare against the actual output of the program.</p> |
| <p>Tests with the <code>*-pass</code> directives can be overridden with the <code>--pass</code> |
| command-line option:</p> |
| <pre><code class="language-sh">./x test tests/ui --pass check |
| </code></pre> |
| <p>The <code>--pass</code> option only affects UI tests. Using <code>--pass check</code> can run the UI |
| test suite much faster (roughly twice as fast on my system), though obviously |
| not exercising as much.</p> |
| <p>The <code>ignore-pass</code> directive can be used to ignore the <code>--pass</code> CLI flag if the |
| test won't work properly with that override.</p> |
| <h2 id="known-bugs"><a class="header" href="#known-bugs">Known bugs</a></h2> |
| <p>The <code>known-bug</code> directive may be used for tests that demonstrate a known bug |
| that has not yet been fixed. Adding tests for known bugs is helpful for several |
| reasons, including:</p> |
| <ol> |
| <li>Maintaining a functional test that can be conveniently reused when the bug is |
| fixed.</li> |
| <li>Providing a sentinel that will fail if the bug is incidentally fixed. This |
| can alert the developer so they know that the associated issue has been fixed |
| and can possibly be closed.</li> |
| </ol> |
| <p>This directive takes comma-separated issue numbers as arguments, or <code>"unknown"</code>:</p> |
| <ul> |
| <li><code>//@ known-bug: #123, #456</code> (when the issues are on rust-lang/rust)</li> |
| <li><code>//@ known-bug: rust-lang/chalk#123456</code> |
| (allows arbitrary text before the <code>#</code>, which is useful when the issue is on another repo)</li> |
| <li><code>//@ known-bug: unknown</code> |
| (when there is no known issue yet; preferably open one if it does not already exist)</li> |
| </ul> |
| <p>Do not include <a href="tests/ui.html#error-annotations">error annotations</a> in a test with |
| <code>known-bug</code>. The test should still include other normal directives and |
| stdout/stderr files.</p> |
| <h2 id="test-organization-1"><a class="header" href="#test-organization-1">Test organization</a></h2> |
| <p>When deciding where to place a test file, please try to find a subdirectory that |
| best matches what you are trying to exercise. Do your best to keep things |
| organized. Admittedly it can be difficult as some tests can overlap different |
| categories, and the existing layout may not fit well.</p> |
| <p>Name the test by a concise description of what the test is checking. Avoid |
| including the issue number in the test name. See <a href="tests/best-practices.html">best |
| practices</a> for a more in-depth discussion of this.</p> |
| <p>Ideally, the test should be added to a directory that helps identify what piece |
| of code is being tested here (e.g., |
| <code>tests/ui/borrowck/reject-move-out-of-borrow-via-pat.rs</code>)</p> |
| <p>When writing a new feature, you may want to <strong>create a subdirectory to store |
| your tests</strong>. For example, if you are implementing RFC 1234 ("Widgets"), then it |
| might make sense to put the tests in a directory like |
| <code>tests/ui/rfc1234-widgets/</code>.</p> |
| <p>In other cases, there may already be a suitable directory.</p> |
| <p>Over time, the <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui"><code>tests/ui</code></a> directory has grown very fast. There is a check in |
| <a href="tests/intro.html#tidy">tidy</a> that will ensure none of the subdirectories has more than |
| 1000 entries. Having too many files causes problems because it isn't editor/IDE |
| friendly and the GitHub UI won't show more than 1000 entries. However, since |
| <code>tests/ui</code> (UI test root directory) and <code>tests/ui/issues</code> directories have more |
| than 1000 entries, we set a different limit for those directories. So, please |
| avoid putting a new test there and try to find a more relevant place.</p> |
| <p>For example, if your test is related to closures, you should put it in |
| <code>tests/ui/closures</code>. When you reach the limit, you could increase it by tweaking |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/tools/tidy/src/ui_tests.rs">here</a>.</p> |
| <h2 id="rustfix-tests"><a class="header" href="#rustfix-tests">Rustfix tests</a></h2> |
| <p>UI tests can validate that diagnostic suggestions apply correctly and that the |
| resulting changes compile correctly. This can be done with the <code>run-rustfix</code> |
| directive:</p> |
| <pre><code class="language-rust ignore">//@ run-rustfix |
| //@ check-pass |
| #![crate_type = "lib"] |
| |
| pub struct not_camel_case {} |
| //~^ WARN `not_camel_case` should have an upper camel case name |
| //~| HELP convert the identifier to upper camel case |
| //~| SUGGESTION NotCamelCase</code></pre> |
| <p>Rustfix tests should have a file with the <code>.fixed</code> extension which contains the |
| source file after the suggestion has been applied.</p> |
| <ul> |
| <li>When the test is run, compiletest first checks that the correct lint/warning |
| is generated.</li> |
| <li>Then, it applies the suggestion and compares against <code>.fixed</code> (they must |
| match).</li> |
| <li>Finally, the fixed source is compiled, and this compilation is required to |
| succeed.</li> |
| </ul> |
| <p>Usually when creating a rustfix test you will generate the <code>.fixed</code> file |
| automatically with the <code>x test --bless</code> option.</p> |
| <p>The <code>run-rustfix</code> directive will cause <em>all</em> suggestions to be applied, even if |
| they are not <a href="tests/../diagnostics.html#suggestions"><code>MachineApplicable</code></a>. If this is a |
| problem, then you can add the <code>rustfix-only-machine-applicable</code> directive in |
| addition to <code>run-rustfix</code>. This should be used if there is a mixture of |
| different suggestion levels, and some of the non-machine-applicable ones do not |
| apply cleanly.</p> |
| <h2 id="compare-modes-1"><a class="header" href="#compare-modes-1">Compare modes</a></h2> |
| <p><a href="tests/compiletest.html#compare-modes">Compare modes</a> can be used to run all tests with |
| different flags from what they are normally compiled with. In some cases, this |
| might result in different output from the compiler. To support this, different |
| output files can be saved which contain the output based on the compare mode.</p> |
| <p>For example, when using the Polonius mode, a test <code>foo.rs</code> will first look for |
| expected output in <code>foo.polonius.stderr</code>, falling back to the usual <code>foo.stderr</code> |
| if not found. This is useful as different modes can sometimes result in |
| different diagnostics and behavior. This can help track which tests have |
| differences between the modes, and to visually inspect those diagnostic |
| differences.</p> |
| <p>If in the rare case you encounter a test that has different behavior, you can |
| run something like the following to generate the alternate stderr file:</p> |
| <pre><code class="language-sh">./x test tests/ui --compare-mode=polonius --bless |
| </code></pre> |
| <p>Currently none of the compare modes are checked in CI for UI tests.</p> |
| <h2 id="rustc_-test-attributes"><a class="header" href="#rustc_-test-attributes"><code>rustc_*</code> TEST attributes</a></h2> |
| <p>The compiler defines several perma-unstable <code>#[rustc_*]</code> attributes gated behind |
| the internal feature <code>rustc_attrs</code> that dump extra compiler-internal |
| information. See the corresponding subsection in <a href="tests/../compiler-debugging.html#rustc_-test-attributes">compiler debugging</a> for more |
| details.</p> |
| <p>They can be used in tests to more precisely, legibly and easily test internal |
| compiler state in cases where it would otherwise be very hard to do the same |
| with "user-facing" Rust alone. Indeed, one could say that this slightly abuses |
| the term "UI" (<em>user</em> interface) and turns such UI tests from black-box tests |
| into white-box ones. Use them carefully and sparingly.</p> |
| <h2 id="ui-test-mode-preset-lint-levels"><a class="header" href="#ui-test-mode-preset-lint-levels">UI test mode preset lint levels</a></h2> |
| <p>By default, test suites under UI test mode (<code>tests/ui</code>, <code>tests/ui-fulldeps</code>, |
| but not <code>tests/rustdoc-ui</code>) will specify</p> |
| <ul> |
| <li><code>-A unused</code></li> |
| <li><code>-A internal_features</code></li> |
| </ul> |
| <p>If:</p> |
| <ul> |
| <li>The ui test's pass mode is below <code>run</code> (i.e. check or build).</li> |
| <li>No compare modes are specified.</li> |
| </ul> |
| <p>Since they can be very noisy in ui tests.</p> |
| <p>You can override them with <code>compile-flags</code> lint level flags or |
| in-source lint level attributes as required.</p> |
| <p>Note that the <code>rustfix</code> version will <em>not</em> have <code>-A unused</code> passed, |
| meaning that you may have to <code>#[allow(unused)]</code> to suppress <code>unused</code> |
| lints on the rustfix'd file (because we might be testing rustfix |
| on <code>unused</code> lints themselves).</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-main test file"> |
| <p>This is a file that has the <code>~?</code> annotations, |
| as distinct from aux files, or sources that we have no control over. <a href="#fr-main test file-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="compiletest-directives-1"><a class="header" href="#compiletest-directives-1">Compiletest directives</a></h1> |
| <!-- |
| FIXME(jieyouxu) completely revise this chapter. |
| --> |
| <p>Directives are special comments that tell compiletest how to build and interpret a test. |
| They may also appear in <code>rmake.rs</code> <a href="tests/compiletest.html#run-make-tests">run-make tests</a>.</p> |
| <p>They are normally put after the short comment that explains the point of this |
| test. Compiletest test suites use <code>//@</code> to signal that a comment is a directive. |
| For example, this test uses the <code>//@ compile-flags</code> command to specify a custom |
| flag to give to rustc when the test is compiled:</p> |
| <pre><code class="language-rust ignore">// Test the behavior of `0 - 1` when overflow checks are disabled. |
| |
| //@ compile-flags: -C overflow-checks=off |
| |
| fn main() { |
| let x = 0 - 1; |
| ... |
| }</code></pre> |
| <p>Directives can be standalone (like <code>//@ run-pass</code>) or take a value (like <code>//@ compile-flags: -C overflow-checks=off</code>).</p> |
| <p>Directives are written one directive per line: you cannot write multiple |
| directives on the same line. For example, if you write <code>//@ only-x86 only-windows</code> then <code>only-windows</code> is interpreted as a comment, not a separate |
| directive.</p> |
| <h2 id="listing-of-compiletest-directives"><a class="header" href="#listing-of-compiletest-directives">Listing of compiletest directives</a></h2> |
| <p>The following is a list of compiletest directives. Directives are linked to |
| sections that describe the command in more detail if available. This list may |
| not be exhaustive. Directives can generally be found by browsing the |
| <code>TestProps</code> structure found in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/directives.rs"><code>directives.rs</code></a> from the compiletest source.</p> |
| <h3 id="assembly"><a class="header" href="#assembly">Assembly</a></h3> |
| <!-- date-check: Oct 2024 --> |
| <div class="table-wrapper"><table><thead><tr><th>Directive</th><th>Explanation</th><th>Supported test suites</th><th>Possible values</th></tr></thead><tbody> |
| <tr><td><code>assembly-output</code></td><td>Assembly output kind to check</td><td><code>assembly</code></td><td><code>emit-asm</code>, <code>bpf-linker</code>, <code>ptx-linker</code></td></tr> |
| </tbody></table> |
| </div> |
| <h3 id="auxiliary-builds"><a class="header" href="#auxiliary-builds">Auxiliary builds</a></h3> |
| <p>See <a href="tests/compiletest.html#building-auxiliary-crates">Building auxiliary crates</a></p> |
| <div class="table-wrapper"><table><thead><tr><th>Directive</th><th>Explanation</th><th>Supported test suites</th><th>Possible values</th></tr></thead><tbody> |
| <tr><td><code>aux-bin</code></td><td>Build a aux binary, made available in <code>auxiliary/bin</code> relative to test directory</td><td>All except <code>run-make</code>/<code>run-make-cargo</code></td><td>Path to auxiliary <code>.rs</code> file</td></tr> |
| <tr><td><code>aux-build</code></td><td>Build a separate crate from the named source file</td><td>All except <code>run-make</code>/<code>run-make-cargo</code></td><td>Path to auxiliary <code>.rs</code> file</td></tr> |
| <tr><td><code>aux-crate</code></td><td>Like <code>aux-build</code> but makes available as extern prelude</td><td>All except <code>run-make</code>/<code>run-make-cargo</code></td><td><code><extern_prelude_name>=<path/to/aux/file.rs></code></td></tr> |
| <tr><td><code>aux-codegen-backend</code></td><td>Similar to <code>aux-build</code> but pass the compiled dylib to <code>-Zcodegen-backend</code> when building the main file</td><td><code>ui-fulldeps</code></td><td>Path to codegen backend file</td></tr> |
| <tr><td><code>proc-macro</code></td><td>Similar to <code>aux-build</code>, but for aux forces host and don't use <code>-Cprefer-dynamic</code><sup class="footnote-reference" id="fr-pm-1"><a href="#footnote-pm">1</a></sup>.</td><td>All except <code>run-make</code>/<code>run-make-cargo</code></td><td>Path to auxiliary proc-macro <code>.rs</code> file</td></tr> |
| <tr><td><code>build-aux-docs</code></td><td>Build docs for auxiliaries as well. Note that this only works with <code>aux-build</code>, not <code>aux-crate</code>.</td><td>All except <code>run-make</code>/<code>run-make-cargo</code></td><td>N/A</td></tr> |
| </tbody></table> |
| </div> |
| <h3 id="controlling-outcome-expectations"><a class="header" href="#controlling-outcome-expectations">Controlling outcome expectations</a></h3> |
| <p>See <a href="tests/ui.html#controlling-passfail-expectations">Controlling pass/fail |
| expectations</a>.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Directive</th><th>Explanation</th><th>Supported test suites</th><th>Possible values</th></tr></thead><tbody> |
| <tr><td><code>check-pass</code></td><td>Building (no codegen) should pass</td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>check-fail</code></td><td>Building (no codegen) should fail</td><td><code>ui</code>, <code>crashes</code></td><td>N/A</td></tr> |
| <tr><td><code>build-pass</code></td><td>Building should pass</td><td><code>ui</code>, <code>crashes</code>, <code>codegen</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>build-fail</code></td><td>Building should fail</td><td><code>ui</code>, <code>crashes</code></td><td>N/A</td></tr> |
| <tr><td><code>run-pass</code></td><td>Program must exit with code <code>0</code></td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>run-fail</code></td><td>Program must exit with code <code>1..=127</code></td><td><code>ui</code>, <code>crashes</code></td><td>N/A</td></tr> |
| <tr><td><code>run-crash</code></td><td>Program must crash</td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>run-fail-or-crash</code></td><td>Program must <code>run-fail</code> or <code>run-crash</code></td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>ignore-pass</code></td><td>Ignore <code>--pass</code> flag</td><td><code>ui</code>, <code>crashes</code>, <code>codegen</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>dont-check-failure-status</code></td><td>Don't check exact failure status (i.e. <code>1</code>)</td><td><code>ui</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>failure-status</code></td><td>Check</td><td><code>ui</code>, <code>crashes</code></td><td>Any <code>u16</code></td></tr> |
| <tr><td><code>should-ice</code></td><td>Check failure status is <code>101</code></td><td><code>coverage</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>should-fail</code></td><td>Compiletest self-test</td><td>All</td><td>N/A</td></tr> |
| </tbody></table> |
| </div> |
| <h3 id="controlling-output-snapshots-and-normalizations"><a class="header" href="#controlling-output-snapshots-and-normalizations">Controlling output snapshots and normalizations</a></h3> |
| <p>See <a href="tests/ui.html#normalization">Normalization</a>, <a href="tests/ui.html#output-comparison">Output |
| comparison</a> and <a href="tests/ui.html#rustfix-tests">Rustfix tests</a> |
| for more details.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Directive</th><th>Explanation</th><th>Supported test suites</th><th>Possible values</th></tr></thead><tbody> |
| <tr><td><code>check-run-results</code></td><td>Check run test binary <code>run-{pass,fail}</code> output snapshot</td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code> if <code>run-pass</code></td><td>N/A</td></tr> |
| <tr><td><code>error-pattern</code></td><td>Check that output contains a specific string</td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code> if <code>run-pass</code></td><td>String</td></tr> |
| <tr><td><code>regex-error-pattern</code></td><td>Check that output contains a regex pattern</td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code> if <code>run-pass</code></td><td>Regex</td></tr> |
| <tr><td><code>check-stdout</code></td><td>Check <code>stdout</code> against <code>error-pattern</code>s from running test binary<sup class="footnote-reference" id="fr-check_stdout-1"><a href="#footnote-check_stdout">2</a></sup></td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code></td><td>N/A</td></tr> |
| <tr><td><code>normalize-stderr-32bit</code></td><td>Normalize actual stderr (for 32-bit platforms) with a rule <code>"<raw>" -> "<normalized>"</code> before comparing against snapshot</td><td><code>ui</code>, <code>incremental</code></td><td><code>"<RAW>" -> "<NORMALIZED>"</code>, <code><RAW></code>/<code><NORMALIZED></code> is regex capture and replace syntax</td></tr> |
| <tr><td><code>normalize-stderr-64bit</code></td><td>Normalize actual stderr (for 64-bit platforms) with a rule <code>"<raw>" -> "<normalized>"</code> before comparing against snapshot</td><td><code>ui</code>, <code>incremental</code></td><td><code>"<RAW>" -> "<NORMALIZED>"</code>, <code><RAW></code>/<code><NORMALIZED></code> is regex capture and replace syntax</td></tr> |
| <tr><td><code>normalize-stderr</code></td><td>Normalize actual stderr with a rule <code>"<raw>" -> "<normalized>"</code> before comparing against snapshot</td><td><code>ui</code>, <code>incremental</code></td><td><code>"<RAW>" -> "<NORMALIZED>"</code>, <code><RAW></code>/<code><NORMALIZED></code> is regex capture and replace syntax</td></tr> |
| <tr><td><code>normalize-stdout</code></td><td>Normalize actual stdout with a rule <code>"<raw>" -> "<normalized>"</code> before comparing against snapshot</td><td><code>ui</code>, <code>incremental</code></td><td><code>"<RAW>" -> "<NORMALIZED>"</code>, <code><RAW></code>/<code><NORMALIZED></code> is regex capture and replace syntax</td></tr> |
| <tr><td><code>dont-check-compiler-stderr</code></td><td>Don't check actual compiler stderr vs stderr snapshot</td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>dont-check-compiler-stdout</code></td><td>Don't check actual compiler stdout vs stdout snapshot</td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>dont-require-annotations</code></td><td>Don't require line annotations for the given diagnostic kind (<code>//~ KIND</code>) to be exhaustive</td><td><code>ui</code>, <code>incremental</code></td><td><code>ERROR</code>, <code>WARN</code>, <code>NOTE</code>, <code>HELP</code>, <code>SUGGESTION</code></td></tr> |
| <tr><td><code>run-rustfix</code></td><td>Apply all suggestions via <code>rustfix</code>, snapshot fixed output, and check fixed output builds</td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>rustfix-only-machine-applicable</code></td><td><code>run-rustfix</code> but only machine-applicable suggestions</td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>exec-env</code></td><td>Env var to set when executing a test</td><td><code>ui</code>, <code>crashes</code></td><td><code><KEY>=<VALUE></code></td></tr> |
| <tr><td><code>unset-exec-env</code></td><td>Env var to unset when executing a test</td><td><code>ui</code>, <code>crashes</code></td><td>Any env var name</td></tr> |
| <tr><td><code>stderr-per-bitwidth</code></td><td>Generate a stderr snapshot for each bitwidth</td><td><code>ui</code></td><td>N/A</td></tr> |
| <tr><td><code>forbid-output</code></td><td>A pattern which must not appear in stderr/<code>cfail</code> output</td><td><code>ui</code>, <code>incremental</code></td><td>Regex pattern</td></tr> |
| <tr><td><code>run-flags</code></td><td>Flags passed to the test executable</td><td><code>ui</code></td><td>Arbitrary flags</td></tr> |
| <tr><td><code>known-bug</code></td><td>No error annotation needed due to known bug</td><td><code>ui</code>, <code>crashes</code>, <code>incremental</code></td><td>Issue number <code>#123456</code></td></tr> |
| <tr><td><code>compare-output-by-lines</code></td><td>Compare the output by lines, rather than as a single string</td><td>All</td><td>N/A</td></tr> |
| </tbody></table> |
| </div> |
| <h3 id="controlling-when-tests-are-run"><a class="header" href="#controlling-when-tests-are-run">Controlling when tests are run</a></h3> |
| <p>These directives are used to ignore the test in some situations, which |
| means the test won't be compiled or run.</p> |
| <ul> |
| <li><code>ignore-X</code> where <code>X</code> is a target detail or other criteria on which to ignore the test (see below)</li> |
| <li><code>only-X</code> is like <code>ignore-X</code>, but will <em>only</em> run the test on that target or |
| stage</li> |
| <li><code>ignore-auxiliary</code> is intended for files that <em>participate</em> in one or more other |
| main test files but that <code>compiletest</code> should not try to build the file itself. |
| Please backlink to which main test is actually using the auxiliary file.</li> |
| <li><code>ignore-test</code> always ignores the test. This can be used to temporarily disable |
| a test if it is currently not working, but you want to keep it in tree to |
| re-enable it later.</li> |
| </ul> |
| <p>Some examples of <code>X</code> in <code>ignore-X</code> or <code>only-X</code>:</p> |
| <ul> |
| <li>A full target triple: <code>aarch64-apple-ios</code></li> |
| <li>Architecture: <code>aarch64</code>, <code>arm</code>, <code>mips</code>, <code>wasm32</code>, <code>x86_64</code>, <code>x86</code>, |
| ...</li> |
| <li>OS: <code>android</code>, <code>emscripten</code>, <code>freebsd</code>, <code>ios</code>, <code>linux</code>, <code>macos</code>, <code>windows</code>, |
| ...</li> |
| <li>Environment (fourth word of the target triple): <code>gnu</code>, <code>msvc</code>, <code>musl</code></li> |
| <li>Pointer width: <code>32bit</code>, <code>64bit</code></li> |
| <li>Endianness: <code>endian-big</code></li> |
| <li>Stage: <code>stage1</code>, <code>stage2</code></li> |
| <li>Binary format: <code>elf</code></li> |
| <li>Channel: <code>stable</code>, <code>beta</code></li> |
| <li>When cross compiling: <code>cross-compile</code></li> |
| <li>When <a href="tests/running.html#running-tests-on-a-remote-machine">remote testing</a> is used: <code>remote</code></li> |
| <li>When particular debuggers are being tested: <code>cdb</code>, <code>gdb</code>, <code>lldb</code></li> |
| <li>When particular debugger versions are matched: <code>ignore-gdb-version</code></li> |
| <li>Specific <a href="tests/ui.html#compare-modes">compare modes</a>: <code>compare-mode-polonius</code>, <code>compare-mode-chalk</code>, |
| <code>compare-mode-split-dwarf</code>, <code>compare-mode-split-dwarf-single</code></li> |
| <li>The two different test modes used by coverage tests: |
| <code>ignore-coverage-map</code>, <code>ignore-coverage-run</code></li> |
| <li>When testing a dist toolchain: <code>dist</code> |
| <ul> |
| <li>This needs to be enabled with <code>COMPILETEST_ENABLE_DIST_TESTS=1</code></li> |
| </ul> |
| </li> |
| <li>The <code>rustc_abi</code> of the target: e.g. <code>rustc_abi-x86_64-sse2</code></li> |
| </ul> |
| <p>The following directives will check rustc build settings and target |
| settings:</p> |
| <ul> |
| <li><code>needs-asm-support</code> — ignores if the <strong>host</strong> architecture doesn't have |
| stable support for <code>asm!</code>. For tests that cross-compile to explicit targets |
| via <code>--target</code>, use <code>needs-llvm-components</code> instead to ensure the appropriate |
| backend is available.</li> |
| <li><code>needs-profiler-runtime</code> — ignores the test if the profiler runtime was not |
| enabled for the target |
| (<code>build.profiler = true</code> in rustc's <code>bootstrap.toml</code>)</li> |
| <li><code>needs-sanitizer-support</code> — ignores if the sanitizer support was not enabled |
| for the target (<code>sanitizers = true</code> in rustc's <code>bootstrap.toml</code>)</li> |
| <li><code>needs-sanitizer-{address,hwaddress,leak,memory,thread}</code> — ignores if the |
| corresponding sanitizer is not enabled for the target (AddressSanitizer, |
| hardware-assisted AddressSanitizer, LeakSanitizer, MemorySanitizer or |
| ThreadSanitizer respectively)</li> |
| <li><code>needs-run-enabled</code> — ignores if it is a test that gets executed, and running |
| has been disabled. Running tests can be disabled with the <code>x test --run=never</code> |
| flag, or running on fuchsia.</li> |
| <li><code>needs-unwind</code> — ignores if the target does not support unwinding</li> |
| <li><code>needs-rust-lld</code> — ignores if the rust lld support is not enabled (<code>rust.lld = true</code> in <code>bootstrap.toml</code>)</li> |
| <li><code>needs-threads</code> — ignores if the target does not have threading support</li> |
| <li><code>needs-subprocess</code> — ignores if the target does not have subprocess support</li> |
| <li><code>needs-symlink</code> — ignores if the target does not support symlinks. This can be |
| the case on Windows if the developer did not enable privileged symlink |
| permissions.</li> |
| <li><code>ignore-std-debug-assertions</code> — ignores if std was built with debug |
| assertions.</li> |
| <li><code>needs-std-debug-assertions</code> — ignores if std was not built with debug |
| assertions.</li> |
| <li><code>ignore-rustc-debug-assertions</code> — ignores if rustc was built with debug |
| assertions.</li> |
| <li><code>needs-rustc-debug-assertions</code> — ignores if rustc was not built with debug |
| assertions.</li> |
| <li><code>needs-target-has-atomic</code> — ignores if target does not have support for all |
| specified atomic widths, e.g. the test with <code>//@ needs-target-has-atomic: 8, 16, ptr</code> will only run if it supports the comma-separated list of atomic |
| widths.</li> |
| <li><code>needs-dynamic-linking</code> — ignores if target does not support dynamic linking |
| (which is orthogonal to it being unable to create <code>dylib</code> and <code>cdylib</code> crate types)</li> |
| <li><code>needs-crate-type</code> — ignores if target platform does not support one or more |
| of the comma-delimited list of specified crate types. For example, |
| <code>//@ needs-crate-type: cdylib, proc-macro</code> will cause the test to be ignored |
| on <code>wasm32-unknown-unknown</code> target because the target does not support the |
| <code>proc-macro</code> crate type.</li> |
| <li><code>needs-target-std</code> — ignores if target platform does not have std support.</li> |
| <li><code>ignore-backends</code> — ignores the listed backends, separated by whitespace characters. Please note |
| that this directive can be overriden with the <code>--bypass-ignore-backends=[BACKEND]</code> command line |
| flag.</li> |
| <li><code>needs-backends</code> — only runs the test if current codegen backend is listed.</li> |
| </ul> |
| <p>The following directives will check LLVM support:</p> |
| <ul> |
| <li><code>exact-llvm-major-version: 19</code> — ignores if the llvm major version does not |
| match the specified llvm major version.</li> |
| <li><code>min-llvm-version: 13.0</code> — ignored if the LLVM version is less than the given |
| value</li> |
| <li><code>min-system-llvm-version: 12.0</code> — ignored if using a system LLVM and its |
| version is less than the given value</li> |
| <li><code>max-llvm-major-version: 19</code> — ignored if the LLVM major version is higher |
| than the given major version</li> |
| <li><code>ignore-llvm-version: 9.0</code> — ignores a specific LLVM version</li> |
| <li><code>ignore-llvm-version: 7.0 - 9.9.9</code> — ignores LLVM versions in a range |
| (inclusive)</li> |
| <li><code>needs-llvm-components: powerpc</code> — ignores if the specific LLVM component was |
| not built. Note: The test will fail on CI (when |
| <code>COMPILETEST_REQUIRE_ALL_LLVM_COMPONENTS</code> is set) if the component does not |
| exist.</li> |
| <li><code>needs-forced-clang-based-tests</code> — test is ignored unless the environment |
| variable <code>RUSTBUILD_FORCE_CLANG_BASED_TESTS</code> is set, which enables building |
| clang alongside LLVM |
| <ul> |
| <li>This is only set in two CI jobs (<a href="https://github.com/rust-lang/rust/blob/ab3dba92db355b8d97db915a2dca161a117e959c/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile#L32"><code>x86_64-gnu-debug</code></a> and |
| <a href="https://github.com/rust-lang/rust/blob/20c909ff9cdd88d33768a4ddb8952927a675b0ad/src/ci/docker/host-aarch64/aarch64-gnu-debug/Dockerfile#L32"><code>aarch64-gnu-debug</code></a>), which only runs a |
| subset of <code>run-make</code> tests. Other tests with this directive will not |
| run at all, which is usually not what you want.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>See also <a href="tests/compiletest.html#debuginfo-tests">Debuginfo tests</a> for directives for |
| ignoring debuggers.</p> |
| <h3 id="affecting-how-tests-are-built"><a class="header" href="#affecting-how-tests-are-built">Affecting how tests are built</a></h3> |
| <div class="table-wrapper"><table><thead><tr><th>Directive</th><th>Explanation</th><th>Supported test suites</th><th>Possible values</th></tr></thead><tbody> |
| <tr><td><code>compile-flags</code></td><td>Flags passed to <code>rustc</code> when building the test or aux file</td><td>All except for <code>run-make</code>/<code>run-make-cargo</code></td><td>Any valid <code>rustc</code> flags, e.g. <code>-Awarnings -Dfoo</code>. Cannot be <code>-Cincremental</code> or <code>--edition</code></td></tr> |
| <tr><td><code>edition</code></td><td>The edition used to build the test</td><td>All except for <code>run-make</code>/<code>run-make-cargo</code></td><td>Any valid <code>--edition</code> value</td></tr> |
| <tr><td><code>rustc-env</code></td><td>Env var to set when running <code>rustc</code></td><td>All except for <code>run-make</code>/<code>run-make-cargo</code></td><td><code><KEY>=<VALUE></code></td></tr> |
| <tr><td><code>unset-rustc-env</code></td><td>Env var to unset when running <code>rustc</code></td><td>All except for <code>run-make</code>/<code>run-make-cargo</code></td><td>Any env var name</td></tr> |
| <tr><td><code>incremental</code></td><td>Proper incremental support for tests outside of incremental test suite</td><td><code>ui</code>, <code>crashes</code></td><td>N/A</td></tr> |
| <tr><td><code>no-prefer-dynamic</code></td><td>Don't use <code>-C prefer-dynamic</code>, don't build as a dylib via a <code>--crate-type=dylib</code> preset flag</td><td><code>ui</code>, <code>crashes</code></td><td>N/A</td></tr> |
| </tbody></table> |
| </div><div class="warning"> |
| <p>Tests (outside of <code>run-make</code>/<code>run-make-cargo</code>) that want to use incremental tests not in the |
| incremental test-suite must not pass <code>-C incremental</code> via <code>compile-flags</code>, and |
| must instead use the <code>//@ incremental</code> directive.</p> |
| <p>Consider writing the test as a proper incremental test instead.</p> |
| </div> |
| <h4 id="the-edition-directive"><a class="header" href="#the-edition-directive">The edition directive</a></h4> |
| <p>The <code>//@ edition</code> directive can take an exact edition, a bounded range of editions, |
| or a left-bounded half-open range of editions. |
| This affects which edition is used by <code>./x test</code> to run the test.</p> |
| <p>For example:</p> |
| <ul> |
| <li>A test with the <code>//@ edition: 2018</code> directive will only run under the 2018 edition.</li> |
| <li>A test with the <code>//@ edition: 2015..2021</code> directive can be run under the 2015, 2018, and 2021 editions. |
| However, CI will only run the test with the lowest edition in the range (which is 2015 in this example).</li> |
| <li>A test with the <code>//@ edition: 2018..</code> directive will run under 2018 edition or greater. |
| However, CI will only run the test with the lowest edition in the range (which is 2018 in this example).</li> |
| </ul> |
| <p>You can also force <code>./x test</code> to use a specific edition by passing the <code>-- --edition=</code> argument. |
| However, tests with the <code>//@ edition</code> directive will clamp the value passed to the argument. |
| For example, if we run <code>./x test -- --edition=2015</code>:</p> |
| <ul> |
| <li>A test with the <code>//@ edition: 2018</code> will run with the 2018 edition.</li> |
| <li>A test with the <code>//@ edition: 2015..2021</code> will be run with the 2015 edition.</li> |
| <li>A test with the <code>//@ edition: 2018..</code> will run with the 2018 edition.</li> |
| </ul> |
| <h3 id="rustdoc"><a class="header" href="#rustdoc">Rustdoc</a></h3> |
| <div class="table-wrapper"><table><thead><tr><th>Directive</th><th>Explanation</th><th>Supported test suites</th><th>Possible values</th></tr></thead><tbody> |
| <tr><td><code>doc-flags</code></td><td>Flags passed to <code>rustdoc</code> when building the test or aux file</td><td><code>rustdoc</code>, <code>rustdoc-js</code>, <code>rustdoc-json</code></td><td>Any valid <code>rustdoc</code> flags</td></tr> |
| </tbody></table> |
| </div><!-- |
| **FIXME(rustdoc)**: what does `check-test-line-numbers-match` do? |
| Asked in |
| <https://rust-lang.zulipchat.com/#narrow/stream/266220-t-rustdoc/topic/What.20is.20the.20.60check-test-line-numbers-match.60.20directive.3F>. |
| --> |
| <h4 id="test-suite-specific-directives"><a class="header" href="#test-suite-specific-directives">Test-suite-specific directives</a></h4> |
| <p>The test suites <a href="tests/../rustdoc-internals/rustdoc-test-suite.html"><code>rustdoc</code></a>, <a href="tests/../rustdoc-internals/search.html#testing-the-search-engine"><code>rustdoc-js</code>/<code>rustdoc-js-std</code></a> |
| and <a href="tests/../rustdoc-internals/rustdoc-json-test-suite.html"><code>rustdoc-json</code></a> each feature an additional set of directives whose basic |
| syntax resembles the one of compiletest directives but which are ultimately read and checked by |
| separate tools. For more information, please read their respective chapters as linked above.</p> |
| <h3 id="pretty-printing"><a class="header" href="#pretty-printing">Pretty printing</a></h3> |
| <p>See <a href="tests/compiletest.html#pretty-printer-tests">Pretty-printer</a>.</p> |
| <h4 id="misc-directives"><a class="header" href="#misc-directives">Misc directives</a></h4> |
| <ul> |
| <li><code>no-auto-check-cfg</code> — disable auto check-cfg (only for <code>--check-cfg</code> tests)</li> |
| <li><a href="tests/compiletest.html#revisions"><code>revisions</code></a> — compile multiple times |
| -<a href="tests/compiletest.html#incremental-tests"><code>forbid-output</code></a> — incremental cfail rejects |
| output pattern</li> |
| <li><a href="tests/compiletest.html#incremental-tests"><code>should-ice</code></a> — incremental cfail should |
| ICE</li> |
| <li><a href="https://github.com/rust-lang/reference/blob/master/docs/authoring.md#test-rule-annotations"><code>reference</code></a> — an annotation linking to a rule in the reference</li> |
| <li><code>disable-gdb-pretty-printers</code> — disable gdb pretty printers for debuginfo tests</li> |
| </ul> |
| <h3 id="tool-specific-directives"><a class="header" href="#tool-specific-directives">Tool-specific directives</a></h3> |
| <p>The following directives affect how certain command-line tools are invoked, in |
| test suites that use those tools:</p> |
| <ul> |
| <li><code>filecheck-flags</code> adds extra flags when running LLVM's <code>FileCheck</code> tool. |
| <ul> |
| <li>Used by <a href="tests/compiletest.html#codegen-tests">codegen tests</a>, |
| <a href="tests/compiletest.html#assembly-tests">assembly tests</a>, and |
| <a href="tests/compiletest.html#mir-opt-tests">MIR-opt tests</a>.</li> |
| </ul> |
| </li> |
| <li><code>llvm-cov-flags</code> adds extra flags when running LLVM's <code>llvm-cov</code> tool. |
| <ul> |
| <li>Used by <a href="tests/compiletest.html#coverage-tests">coverage tests</a> in <code>coverage-run</code> mode.</li> |
| </ul> |
| </li> |
| </ul> |
| <h3 id="tidy-specific-directives"><a class="header" href="#tidy-specific-directives">Tidy specific directives</a></h3> |
| <p>The following directives control how the <a href="tests/../conventions.html#formatting">tidy script</a> |
| verifies tests.</p> |
| <ul> |
| <li><code>ignore-tidy-target-specific-tests</code> disables checking that the appropriate |
| LLVM component is required (via a <code>needs-llvm-components</code> directive) when a |
| test is compiled for a specific target (via the <code>--target</code> flag in a |
| <code>compile-flag</code> directive).</li> |
| <li><a href="tests/compiletest.html#ignoring-unused-revision-names"><code>unused-revision-names</code></a> - |
| suppress tidy checks for mentioning unknown revision names.</li> |
| </ul> |
| <h2 id="substitutions"><a class="header" href="#substitutions">Substitutions</a></h2> |
| <p>Directive values support substituting a few variables which will be replaced |
| with their corresponding value. For example, if you need to pass a compiler flag |
| with a path to a specific file, something like the following could work:</p> |
| <pre><code class="language-rust ignore">//@ compile-flags: --remap-path-prefix={{src-base}}=/the/src</code></pre> |
| <p>Where the sentinel <code>{{src-base}}</code> will be replaced with the appropriate path |
| described below:</p> |
| <ul> |
| <li><code>{{cwd}}</code>: The directory where compiletest is run from. This may not be the |
| root of the checkout, so you should avoid using it where possible. |
| <ul> |
| <li>Examples: <code>/path/to/rust</code>, <code>/path/to/build/root</code></li> |
| </ul> |
| </li> |
| <li><code>{{src-base}}</code>: The directory where the test is defined. This is equivalent to |
| <code>$DIR</code> for <a href="tests/ui.html#normalization">output normalization</a>. |
| <ul> |
| <li>Example: <code>/path/to/rust/tests/ui/error-codes</code></li> |
| </ul> |
| </li> |
| <li><code>{{build-base}}</code>: The base directory where the test's output goes. This is |
| equivalent to <code>$TEST_BUILD_DIR</code> for <a href="tests/ui.html#normalization">output normalization</a>. |
| <ul> |
| <li>Example: <code>/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui</code></li> |
| </ul> |
| </li> |
| <li><code>{{rust-src-base}}</code>: The sysroot directory where libstd/libcore/... are |
| located</li> |
| <li><code>{{sysroot-base}}</code>: Path of the sysroot directory used to build the test. |
| <ul> |
| <li>Mainly intended for <code>ui-fulldeps</code> tests that run the compiler via API.</li> |
| </ul> |
| </li> |
| <li><code>{{target-linker}}</code>: Linker that would be passed to <code>-Clinker</code> for this test, |
| or blank if no linker override is active. |
| <ul> |
| <li>Mainly intended for <code>ui-fulldeps</code> tests that run the compiler via API.</li> |
| </ul> |
| </li> |
| <li><code>{{target}}</code>: The target the test is compiling for |
| <ul> |
| <li>Example: <code>x86_64-unknown-linux-gnu</code></li> |
| </ul> |
| </li> |
| </ul> |
| <p>See |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui/argfile/commandline-argfile.rs"><code>tests/ui/argfile/commandline-argfile.rs</code></a> |
| for an example of a test that uses this substitution.</p> |
| <h2 id="adding-a-directive"><a class="header" href="#adding-a-directive">Adding a directive</a></h2> |
| <p>One would add a new directive if there is a need to define some test property or |
| behavior on an individual, test-by-test basis. A directive property serves as |
| the directive's backing store (holds the command's current value) at runtime.</p> |
| <p>To add a new directive property:</p> |
| <ol> |
| <li>Look for the <code>pub struct TestProps</code> declaration in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/directives.rs"><code>src/tools/compiletest/src/directives.rs</code></a> and add the new public property to |
| the end of the declaration.</li> |
| <li>Look for the <code>impl TestProps</code> implementation block immediately following the |
| struct declaration and initialize the new property to its default value.</li> |
| </ol> |
| <h3 id="adding-a-new-directive-parser"><a class="header" href="#adding-a-new-directive-parser">Adding a new directive parser</a></h3> |
| <p>When <code>compiletest</code> encounters a test file, it parses the file a line at a time |
| by calling every parser defined in the <code>Config</code> struct's implementation block, |
| also in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/directives.rs"><code>src/tools/compiletest/src/directives.rs</code></a> (note that the <code>Config</code> struct's |
| declaration block is found in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/common.rs"><code>src/tools/compiletest/src/common.rs</code></a>). |
| <code>TestProps</code>'s <code>load_from()</code> method will try passing the current line of text to |
| each parser, which, in turn typically checks to see if the line begins with a |
| particular commented (<code>//@</code>) directive such as <code>//@ must-compile-successfully</code> |
| or <code>//@ failure-status</code>. Whitespace after the comment marker is optional.</p> |
| <p>Parsers will override a given directive property's default value merely by being |
| specified in the test file as a directive or by having a parameter value |
| specified in the test file, depending on the directive.</p> |
| <p>Parsers defined in <code>impl Config</code> are typically named <code>parse_<directive-name></code> |
| (note kebab-case <code><directive-command></code> transformed to snake-case |
| <code><directive_command></code>). <code>impl Config</code> also defines several 'low-level' parsers |
| which make it simple to parse common patterns like simple presence or not |
| (<code>parse_name_directive()</code>), <code>directive:parameter(s)</code> |
| (<code>parse_name_value_directive()</code>), optional parsing only if a particular <code>cfg</code> |
| attribute is defined (<code>has_cfg_prefix()</code>) and many more. The low-level parsers |
| are found near the end of the <code>impl Config</code> block; be sure to look through them |
| and their associated parsers immediately above to see how they are used to avoid |
| writing additional parsing code unnecessarily.</p> |
| <p>As a concrete example, here is the implementation for the |
| <code>parse_failure_status()</code> parser, in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/directives.rs"><code>src/tools/compiletest/src/directives.rs</code></a>:</p> |
| <pre><code class="language-diff">@@ -232,6 +232,7 @@ pub struct TestProps { |
| // customized normalization rules |
| pub normalize_stdout: Vec<(String, String)>, |
| pub normalize_stderr: Vec<(String, String)>, |
| + pub failure_status: i32, |
| } |
| |
| impl TestProps { |
| @@ -260,6 +261,7 @@ impl TestProps { |
| run_pass: false, |
| normalize_stdout: vec![], |
| normalize_stderr: vec![], |
| + failure_status: 101, |
| } |
| } |
| |
| @@ -383,6 +385,10 @@ impl TestProps { |
| if let Some(rule) = config.parse_custom_normalization(ln, "normalize-stderr") { |
| self.normalize_stderr.push(rule); |
| } |
| + |
| + if let Some(code) = config.parse_failure_status(ln) { |
| + self.failure_status = code; |
| + } |
| }); |
| |
| for key in &["RUST_TEST_NOCAPTURE", "RUST_TEST_THREADS"] { |
| @@ -488,6 +494,13 @@ impl Config { |
| self.parse_name_directive(line, "pretty-compare-only") |
| } |
| |
| + fn parse_failure_status(&self, line: &str) -> Option<i32> { |
| + match self.parse_name_value_directive(line, "failure-status") { |
| + Some(code) => code.trim().parse::<i32>().ok(), |
| + _ => None, |
| + } |
| + } |
| </code></pre> |
| <h3 id="implementing-the-behavior-change"><a class="header" href="#implementing-the-behavior-change">Implementing the behavior change</a></h3> |
| <p>When a test invokes a particular directive, it is expected that some behavior |
| will change as a result. What behavior, obviously, will depend on the purpose of |
| the directive. In the case of <code>failure-status</code>, the behavior that changes is |
| that <code>compiletest</code> expects the failure code defined by the directive invoked in |
| the test, rather than the default value.</p> |
| <p>Although specific to <code>failure-status</code> (as every directive will have a different |
| implementation in order to invoke behavior change) perhaps it is helpful to see |
| the behavior change implementation of one case, simply as an example. To |
| implement <code>failure-status</code>, the <code>check_correct_failure_status()</code> function found |
| in the <code>TestCx</code> implementation block, located in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest/src/runtest.rs"><code>src/tools/compiletest/src/runtest.rs</code></a>, was modified as per below:</p> |
| <pre><code class="language-diff">@@ -295,11 +295,14 @@ impl<'test> TestCx<'test> { |
| } |
| |
| fn check_correct_failure_status(&self, proc_res: &ProcRes) { |
| - // The value the Rust runtime returns on failure |
| - const RUST_ERR: i32 = 101; |
| - if proc_res.status.code() != Some(RUST_ERR) { |
| + let expected_status = Some(self.props.failure_status); |
| + let received_status = proc_res.status.code(); |
| + |
| + if expected_status != received_status { |
| self.fatal_proc_rec( |
| - &format!("failure produced the wrong error: {}", proc_res.status), |
| + &format!("Error: expected failure status ({:?}) but received status {:?}.", |
| + expected_status, |
| + received_status), |
| proc_res, |
| ); |
| } |
| @@ -320,7 +323,6 @@ impl<'test> TestCx<'test> { |
| ); |
| |
| let proc_res = self.exec_compiled_test(); |
| - |
| if !proc_res.status.success() { |
| self.fatal_proc_rec("test run failed!", &proc_res); |
| } |
| @@ -499,7 +501,6 @@ impl<'test> TestCx<'test> { |
| expected, |
| actual |
| ); |
| - panic!(); |
| } |
| } |
| </code></pre> |
| <p>Note the use of <code>self.props.failure_status</code> to access the directive property. In |
| tests which do not specify the failure status directive, |
| <code>self.props.failure_status</code> will evaluate to the default value of 101 at the |
| time of this writing. But for a test which specifies a directive of, for |
| example, <code>//@ failure-status: 1</code>, <code>self.props.failure_status</code> will evaluate to |
| 1, as <code>parse_failure_status()</code> will have overridden the <code>TestProps</code> default |
| value, for that test specifically.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-pm"> |
| <p>please see the <a href="tests/compiletest.html#auxiliary-proc-macro">Auxiliary proc-macro section</a> in the compiletest chapter for specifics. <a href="#fr-pm-1">↩</a></p> |
| </li> |
| <li id="footnote-check_stdout"> |
| <p>presently <!-- date-check: Oct 2024 --> this has a weird quirk |
| where the test binary's stdout and stderr gets concatenated and then |
| <code>error-pattern</code>s are matched on this combined output, which is ??? slightly |
| questionable to say the least. <a href="#fr-check_stdout-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="minicore-test-auxiliary-using-core-stubs"><a class="header" href="#minicore-test-auxiliary-using-core-stubs"><code>minicore</code> test auxiliary: using <code>core</code> stubs</a></h1> |
| <!-- date-check: Oct 2025 --> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/tests/auxiliary/minicore.rs"><code>tests/auxiliary/minicore.rs</code></a> is a test auxiliary for ui/codegen/assembly test suites. |
| It provides <code>core</code> stubs for tests that need to |
| build for cross-compiled targets but do not need/want to run.</p> |
| <div class="warning"> |
| <p>Please note that <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/auxiliary/minicore.rs"><code>minicore</code></a> is only intended for <code>core</code> items, and explicitly |
| <strong>not</strong> <code>std</code> or <code>alloc</code> items because <code>core</code> items are applicable to a wider range of tests.</p> |
| </div> |
| <p>A test can use <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/auxiliary/minicore.rs"><code>minicore</code></a> by specifying the <code>//@ add-minicore</code> directive. |
| Then, mark the test with <code>#![feature(no_core)]</code> + <code>#![no_std]</code> + <code>#![no_core]</code>, |
| and import the crate into the test with <code>extern crate minicore</code> (edition 2015) |
| or <code>use minicore</code> (edition 2018+).</p> |
| <h2 id="implied-compiler-flags"><a class="header" href="#implied-compiler-flags">Implied compiler flags</a></h2> |
| <p>Due to the <code>no_std</code> + <code>no_core</code> nature of these tests, <code>//@ add-minicore</code> |
| implies and requires that the test will be built with <code>-C panic=abort</code>. |
| <strong>Unwinding panics are not supported.</strong></p> |
| <p>Tests will also be built with <code>-C force-unwind-tables=yes</code> to preserve CFI |
| directives in assembly tests.</p> |
| <p>TL;DR: <code>//@ add-minicore</code> implies two compiler flags:</p> |
| <ol> |
| <li><code>-C panic=abort</code></li> |
| <li><code>-C force-unwind-tables=yes</code></li> |
| </ol> |
| <h2 id="adding-more-core-stubs"><a class="header" href="#adding-more-core-stubs">Adding more <code>core</code> stubs</a></h2> |
| <p>If you find a <code>core</code> item to be missing from the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/auxiliary/minicore.rs"><code>minicore</code></a> stub, consider |
| adding it to the test auxiliary if it's likely to be used or is already needed |
| by more than one test.</p> |
| <h2 id="staying-in-sync-with-core"><a class="header" href="#staying-in-sync-with-core">Staying in sync with <code>core</code></a></h2> |
| <p>The <code>minicore</code> items must be kept up to date with <code>core</code>. |
| For consistent diagnostic output between using <code>core</code> and <code>minicore</code>, any <code>diagnostic</code> |
| attributes (e.g. <code>on_unimplemented</code>) should be replicated exactly in <code>minicore</code>.</p> |
| <h2 id="example-codegen-test-that-uses-minicore"><a class="header" href="#example-codegen-test-that-uses-minicore">Example codegen test that uses <code>minicore</code></a></h2> |
| <pre><pre class="playground"><code class="language-rust no_run"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>//@ add-minicore |
| //@ revisions: meow bark |
| //@[meow] compile-flags: --target=x86_64-unknown-linux-gnu |
| //@[meow] needs-llvm-components: x86 |
| //@[bark] compile-flags: --target=wasm32-unknown-unknown |
| //@[bark] needs-llvm-components: webassembly |
| |
| #![crate_type = "lib"] |
| #![feature(no_core)] |
| #![no_std] |
| #![no_core] |
| |
| extern crate minicore; |
| use minicore::*; |
| |
| struct Meow; |
| impl Copy for Meow {} // `Copy` here is provided by `minicore` |
| |
| // CHECK-LABEL: meow |
| #[unsafe(no_mangle)] |
| fn meow() {} |
| <span class="boring">}</span></code></pre></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="ecosystem-testing-1"><a class="header" href="#ecosystem-testing-1">Ecosystem testing</a></h1> |
| <p>Rust tests integration with real-world code in the ecosystem to catch |
| regressions and make informed decisions about the evolution of the language.</p> |
| <h2 id="testing-methods"><a class="header" href="#testing-methods">Testing methods</a></h2> |
| <h3 id="crater"><a class="header" href="#crater">Crater</a></h3> |
| <p>Crater is a tool which runs tests on many thousands of public projects. This |
| tool has its own separate infrastructure for running, and is not run as part of |
| CI. See the <a href="tests/crater.html">Crater chapter</a> for more details.</p> |
| <h3 id="cargotest"><a class="header" href="#cargotest"><code>cargotest</code></a></h3> |
| <p><code>cargotest</code> is a small tool which runs <code>cargo test</code> on a few sample projects |
| (such as <code>servo</code>, <code>ripgrep</code>, <code>tokei</code>, etc.). This runs as part of CI and ensures |
| there aren't any significant regressions:</p> |
| <pre><code class="language-console">./x test src/tools/cargotest |
| </code></pre> |
| <h3 id="large-oss-project-builders"><a class="header" href="#large-oss-project-builders">Large OSS Project builders</a></h3> |
| <p>We have CI jobs that build large open-source Rust projects that are used as |
| regression tests in CI. Our integration jobs build the following projects:</p> |
| <ul> |
| <li><a href="tests/./ecosystem-test-jobs/fuchsia.html">Fuchsia</a></li> |
| <li><a href="tests/./ecosystem-test-jobs/rust-for-linux.html">Rust for Linux</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="crater-1"><a class="header" href="#crater-1">Crater</a></h1> |
| <p><a href="https://github.com/rust-lang/crater">Crater</a> is a tool for compiling and |
| running tests for <em>every</em> crate on <a href="https://crates.io">crates.io</a> (and a few on |
| GitHub). It is mainly used for checking the extent of breakage when implementing |
| potentially breaking changes and ensuring lack of breakage by running beta vs |
| stable compiler versions.</p> |
| <h2 id="when-to-run-crater"><a class="header" href="#when-to-run-crater">When to run Crater</a></h2> |
| <p>You should request a Crater run if your PR makes large changes to the compiler |
| or could cause breakage. If you are unsure, feel free to ask your PR's reviewer.</p> |
| <h2 id="requesting-crater-runs"><a class="header" href="#requesting-crater-runs">Requesting Crater Runs</a></h2> |
| <p>The Rust team maintains a few machines that can be used for Crater runs |
| on the changes introduced by a PR. If your PR needs a Crater run, leave a |
| comment for the triage team in the PR thread. Please inform the team whether you |
| require a "check-only" Crater run, a "build only" Crater run, or a |
| "build-and-test" Crater run. The difference is primarily in time; |
| if you're not sure, go for the build-and-test run. If |
| making changes that will only have an effect at compile-time (e.g., implementing |
| a new trait), then you only need a check run.</p> |
| <p>Your PR will be enqueued by the triage team and the results will be posted when |
| they are ready. Check runs will take around ~3-4 days, and the other two taking |
| 5-6 days on average.</p> |
| <p>While Crater is really useful, it is also important to be aware of a few |
| caveats:</p> |
| <ul> |
| <li> |
| <p>Not all code is on crates.io! There is a lot of code in repos on GitHub and |
| elsewhere. Also, companies may not wish to publish their code. Thus, a |
| successful Crater run does not mean there will be no |
| breakage; you still need to be careful.</p> |
| </li> |
| <li> |
| <p>Crater only runs Linux builds on x86_64. Thus, other architectures and |
| platforms are not tested. Critically, this includes Windows.</p> |
| </li> |
| <li> |
| <p>Many crates are not tested. This could be for a lot of reasons, including that |
| the crate doesn't compile any more (e.g. used old nightly features), has |
| broken or flaky tests, requires network access, or other reasons.</p> |
| </li> |
| <li> |
| <p>Before Crater can be run, <code>@bors try</code> needs to succeed in building artifacts. |
| This means that if your code doesn't compile, you cannot run Crater.</p> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="fuchsia-integration-tests"><a class="header" href="#fuchsia-integration-tests">Fuchsia integration tests</a></h1> |
| <p><a href="https://fuchsia.dev">Fuchsia</a> is an open-source operating system with about 2 |
| million lines of Rust code.<sup class="footnote-reference" id="fr-loc-1"><a href="#footnote-loc">1</a></sup> It has caught a large number of <a href="https://gist.github.com/tmandry/7103eba4bd6a6fb0c439b5a90ae355fa">regressions</a> |
| in the past and was subsequently included in CI.</p> |
| <h2 id="what-to-do-if-the-fuchsia-job-breaks"><a class="header" href="#what-to-do-if-the-fuchsia-job-breaks">What to do if the Fuchsia job breaks?</a></h2> |
| <p>Please contact the <a href="tests/ecosystem-test-jobs/../../notification-groups/fuchsia.html">fuchsia</a> ping group and ask them for help.</p> |
| <pre><code class="language-text">@rustbot ping fuchsia |
| </code></pre> |
| <h2 id="building-fuchsia-in-ci"><a class="header" href="#building-fuchsia-in-ci">Building Fuchsia in CI</a></h2> |
| <p>Fuchsia builds as part of the suite of bors tests that run before a pull request |
| is merged.</p> |
| <p>If you are worried that a pull request might break the Fuchsia builder and want |
| to test it out before submitting it to the bors queue, simply ask bors to run |
| the try job that builds the Fuchsia integration: <code>@bors try jobs=x86_64-fuchsia</code>.</p> |
| <h2 id="building-fuchsia-locally"><a class="header" href="#building-fuchsia-locally">Building Fuchsia locally</a></h2> |
| <p>Because Fuchsia uses languages other than Rust, it does not use Cargo as a build |
| system. It also requires the toolchain build to be configured in a <a href="https://fuchsia.dev/fuchsia-src/development/build/rust_toolchain">certain |
| way</a>.</p> |
| <p>The recommended way to build Fuchsia is to use the Docker scripts that check out |
| and run a Fuchsia build for you. If you've run Docker tests before, you can |
| simply run this command from your Rust checkout to download and build Fuchsia |
| using your local Rust toolchain.</p> |
| <pre><code>src/ci/docker/run.sh x86_64-fuchsia |
| </code></pre> |
| <p>See the <a href="tests/ecosystem-test-jobs/../docker.html">Testing with Docker</a> chapter for more details on how to run |
| and debug jobs with Docker.</p> |
| <p>Note that a Fuchsia checkout is <em>large</em> – as of this writing, a checkout and |
| build takes 46G of space – and as you might imagine, it takes a while to |
| complete.</p> |
| <h3 id="modifying-the-fuchsia-checkout"><a class="header" href="#modifying-the-fuchsia-checkout">Modifying the Fuchsia checkout</a></h3> |
| <p>The main reason you would want to build Fuchsia locally is because you need to |
| investigate a regression. After running a Docker build, you'll find the Fuchsia |
| checkout inside the <code>obj/fuchsia</code> directory of your Rust checkout. If you |
| modify the <code>KEEP_CHECKOUT</code> line in the <a href="https://github.com/rust-lang/rust/blob/221e2741c39515a5de6da42d8c76ee1e132c2c74/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh">build-fuchsia.sh</a> script to |
| <code>KEEP_CHECKOUT=1</code>, you can change the checkout as needed and rerun the build |
| command above. This will reuse all the build results from before.</p> |
| <p>You can find more options to customize the Fuchsia checkout in the |
| <a href="https://github.com/rust-lang/rust/blob/221e2741c39515a5de6da42d8c76ee1e132c2c74/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh">build-fuchsia.sh</a> script.</p> |
| <h3 id="customizing-the-fuchsia-build"><a class="header" href="#customizing-the-fuchsia-build">Customizing the Fuchsia build</a></h3> |
| <p>You can find more info about the options used to build Fuchsia in Rust CI in the |
| <a href="https://cs.opensource.google/fuchsia/fuchsia/+/main:scripts/rust/build_fuchsia_from_rust_ci.sh?q=build_fuchsia_from_rust_ci&ss=fuchsia">build_fuchsia_from_rust_ci.sh</a> script invoked by <a href="https://github.com/rust-lang/rust/blob/221e2741c39515a5de6da42d8c76ee1e132c2c74/src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh">build-fuchsia.sh</a>.</p> |
| <p>The Fuchsia build system uses <a href="https://gn.googlesource.com/gn/+/main#gn">GN</a>, a metabuild system that generates <a href="https://ninja-build.org/">Ninja</a> |
| files and then hands off the work of running the build to Ninja.</p> |
| <p>Fuchsia developers use <code>fx</code> to run builds and perform other development tasks. |
| This tool is located in <code>.jiri_root/bin</code> of the Fuchsia checkout; you may need |
| to add this to your <code>$PATH</code> for some workflows.</p> |
| <p>There are a few <code>fx</code> subcommands that are relevant, including:</p> |
| <ul> |
| <li><code>fx set</code> accepts build arguments, writes them to <code>out/default/args.gn</code>, and |
| runs GN.</li> |
| <li><code>fx build</code> builds the Fuchsia project using Ninja. It will automatically pick |
| up changes to build arguments and rerun GN. By default it builds everything, |
| but it also accepts target paths to build specific targets (see below).</li> |
| <li><code>fx clippy</code> runs Clippy on specific Rust targets (or all of them). We use this |
| in the Rust CI build to avoid running codegen on most Rust targets. Underneath |
| it invokes Ninja, just like <code>fx build</code>. The clippy results are saved in json |
| files inside the build output directory before being printed.</li> |
| </ul> |
| <h4 id="target-paths"><a class="header" href="#target-paths">Target paths</a></h4> |
| <p>GN uses paths like the following to identify build targets:</p> |
| <pre><code>//src/starnix/kernel:starnix_core |
| </code></pre> |
| <p>The initial <code>//</code> means the root of the checkout, and the remaining slashes are |
| directory names. The string after <code>:</code> is the <em>target name</em> of a target defined |
| in the <code>BUILD.gn</code> file of that directory.</p> |
| <p>The target name can be omitted if it is the same as the directory name. In other |
| words, <code>//src/starnix/kernel</code> is the same as <code>//src/starnix/kernel:kernel</code>.</p> |
| <p>These target paths are used inside <code>BUILD.gn</code> files to reference dependencies, |
| and can also be used in <code>fx build</code>.</p> |
| <h4 id="modifying-compiler-flags"><a class="header" href="#modifying-compiler-flags">Modifying compiler flags</a></h4> |
| <p>You can put custom compiler flags inside a GN <code>config</code> that is added to a |
| target. As a simple example:</p> |
| <pre><code>config("everybody_loops") { |
| rustflags = [ "-Zeverybody-loops" ] |
| } |
| |
| rustc_binary("example") { |
| crate_root = "src/bin.rs" |
| # ...existing keys here... |
| configs += [ ":everybody_loops" ] |
| } |
| </code></pre> |
| <p>This will add the flag <code>-Zeverybody-loops</code> to rustc when building the <code>example</code> |
| target. Note that you can also use <a href="https://gn.googlesource.com/gn/+/main/docs/reference.md#var_public_configs"><code>public_configs</code></a> for a config to be added |
| to every target that depends on that target.</p> |
| <p>If you want to add a flag to every Rust target in the build, you can add |
| rustflags to the <a href="https://cs.opensource.google/fuchsia/fuchsia/+/main:build/config/BUILD.gn;l=121;drc=c26c473bef93b33117ae417893118907a026fec7"><code>//build/config:compiler</code></a> config or to the OS-specific |
| configs referenced in that file. Note that <code>cflags</code> and <code>ldflags</code> are ignored on |
| Rust targets.</p> |
| <h4 id="running-ninja-and-rustc-commands-directly"><a class="header" href="#running-ninja-and-rustc-commands-directly">Running ninja and rustc commands directly</a></h4> |
| <p>Going down one layer, <code>fx build</code> invokes <code>ninja</code>, which in turn eventually |
| invokes <code>rustc</code>. All build actions are run inside the out directory, which is |
| usually <code>out/default</code> inside the Fuchsia checkout.</p> |
| <p>You can get ninja to print the actual command it invokes by forcing that command |
| to fail, e.g. by adding a syntax error to one of the source files of the target. |
| Once you have the command, you can run it from inside the output directory.</p> |
| <p>After changing the toolchain itself, the build setting <code>rustc_version_string</code> in |
| <code>out/default/args.gn</code> needs to be changed so that <code>fx build</code> or <code>ninja</code> will |
| rebuild all the Rust targets. This can be done in a text editor and the contents |
| of the string do not matter, as long as it changes from one build to the next. |
| <a href="https://cs.opensource.google/fuchsia/fuchsia/+/main:scripts/rust/build_fuchsia_from_rust_ci.sh?q=build_fuchsia_from_rust_ci&ss=fuchsia">build_fuchsia_from_rust_ci.sh</a> does this for you by hashing the toolchain |
| directory.</p> |
| <p>The Fuchsia website has more detailed documentation of the <a href="https://fuchsia.dev/fuchsia-src/development/build/build_system">build system</a>.</p> |
| <h4 id="other-tips-and-tricks"><a class="header" href="#other-tips-and-tricks">Other tips and tricks</a></h4> |
| <p>When using <code>build_fuchsia_from_rust_ci.sh</code> you can comment out the <code>fx set</code> |
| command after the initial run so it won't rerun GN each time. If you do this you |
| can also comment out the version_string line to save a couple seconds.</p> |
| <p><code>export NINJA_PERSISTENT_MODE=1</code> to get faster ninja startup times after the |
| initial build.</p> |
| <h2 id="fuchsia-target-support"><a class="header" href="#fuchsia-target-support">Fuchsia target support</a></h2> |
| <p>To learn more about Fuchsia target support, see the Fuchsia chapter in <a href="https://doc.rust-lang.org/nightly/rustc/platform-support/fuchsia.html">the |
| rustc book</a>.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-loc"> |
| <p>As of June 2024, Fuchsia had about 2 million lines of first-party Rust |
| code and a roughly equal amount of third-party code, as counted by tokei |
| (excluding comments and blanks). <a href="#fr-loc-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="rust-for-linux-integration-tests"><a class="header" href="#rust-for-linux-integration-tests">Rust for Linux integration tests</a></h1> |
| <p><a href="https://rust-for-linux.com/">Rust for Linux</a> (RfL) is an effort for adding |
| support for the Rust programming language into the Linux kernel.</p> |
| <h2 id="what-to-do-if-the-rust-for-linux-job-breaks"><a class="header" href="#what-to-do-if-the-rust-for-linux-job-breaks">What to do if the Rust for Linux job breaks?</a></h2> |
| <p>If a PR breaks the Rust for Linux CI job, then:</p> |
| <ul> |
| <li>If the breakage was unintentional and seems spurious, then let <a href="tests/ecosystem-test-jobs/../../notification-groups/rust-for-linux.html">RfL</a> |
| know and retry. |
| <ul> |
| <li>If the PR is urgent and retrying doesn't fix it, then disable the CI job |
| temporarily (comment out the <code>image: x86_64-rust-for-linux</code> job in |
| <code>src/ci/github-actions/jobs.yml</code>).</li> |
| </ul> |
| </li> |
| <li>If the breakage was unintentional, then change the PR to resolve the breakage.</li> |
| <li>If the breakage was intentional, then let <a href="tests/ecosystem-test-jobs/../../notification-groups/rust-for-linux.html">RfL</a> know and discuss |
| what will the kernel need to change. |
| <ul> |
| <li>If the PR is urgent, then disable the CI job temporarily (comment out |
| the <code>image: x86_64-rust-for-linux</code> job in <code>src/ci/github-actions/jobs.yml</code>).</li> |
| <li>If the PR can wait a few days, then wait for RfL maintainers to provide a |
| new Linux kernel commit hash with the needed changes done, and apply it to |
| the PR, which would confirm the changes work (update the <code>LINUX_VERSION</code> |
| environment variable in <code>src/ci/docker/scripts/rfl-build.sh</code>).</li> |
| </ul> |
| </li> |
| </ul> |
| <p>If you need to contact the RfL developers, you can ping the <a href="tests/ecosystem-test-jobs/../../notification-groups/rust-for-linux.html">Rust for Linux</a> |
| ping group to ask for help:</p> |
| <pre><code class="language-text">@rustbot ping rfl |
| </code></pre> |
| <h2 id="building-rust-for-linux-in-ci"><a class="header" href="#building-rust-for-linux-in-ci">Building Rust for Linux in CI</a></h2> |
| <p>Rust for Linux builds as part of the suite of bors tests that run before a pull |
| request is merged.</p> |
| <p>The workflow builds a stage1 sysroot of the Rust compiler, downloads the Linux |
| kernel, and tries to compile several Rust for Linux drivers and examples using |
| this sysroot. RfL uses several unstable compiler/language features, therefore |
| this workflow notifies us if a given compiler change would break it.</p> |
| <p>If you are worried that a pull request might break the Rust for Linux builder |
| and want to test it out before submitting it to the bors queue, simply ask |
| bors to run the try job that builds the Rust for Linux integration: |
| <code>@bors try jobs=x86_64-rust-for-linux</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="codegen-backend-testing-1"><a class="header" href="#codegen-backend-testing-1">Codegen backend testing</a></h1> |
| <p>See also the <a href="tests/codegen-backend-tests/../../backend/codegen.html">Code generation</a> chapter.</p> |
| <p>In addition to the primary LLVM codegen backend, the rust-lang/rust CI also runs tests of the <a href="https://github.com/rust-lang/rustc_codegen_cranelift">cranelift</a> and <a href="https://github.com/rust-lang/rustc_codegen_gcc">GCC</a> codegen backends in certain test jobs.</p> |
| <p>For more details on the tests involved, see:</p> |
| <ul> |
| <li><a href="tests/codegen-backend-tests/./cg_clif.html">Cranelift codegen backend tests</a></li> |
| <li><a href="tests/codegen-backend-tests/./cg_gcc.html">GCC codegen backend tests</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="cranelift-codegen-backend-tests"><a class="header" href="#cranelift-codegen-backend-tests">Cranelift codegen backend tests</a></h1> |
| <p>TODO: please add some more information to this page.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="gcc-codegen-backend"><a class="header" href="#gcc-codegen-backend">GCC codegen backend</a></h1> |
| <p>We run a subset of the compiler test suite with the GCC codegen backend on our CI, to help find changes that could break the integration of this backend with the compiler.</p> |
| <p>If you encounter any bugs or problems with the GCC codegen backend in general, don't hesitate to open issues on the |
| <a href="https://github.com/rust-lang/rustc_codegen_gcc"><code>rustc_codegen_gcc</code> repository</a>.</p> |
| <p>Note that the backend currently only supports the <code>x86_64-unknown-linux-gnu</code> target.</p> |
| <h2 id="running-into-gcc-backend-ci-errors"><a class="header" href="#running-into-gcc-backend-ci-errors">Running into GCC backend CI errors</a></h2> |
| <p>If you ran into an error related to tests executed with the GCC codegen backend on CI in the <code>x86_64-gnu-gcc</code> job, |
| you can use the following command to run UI tests locally using the GCC backend, which reproduces what happens on CI:</p> |
| <pre><code class="language-bash">./x test tests/ui \ |
| --set 'rust.codegen-backends = ["llvm", "gcc"]' \ |
| --set 'rust.debug-assertions = false' \ |
| --test-codegen-backend gcc |
| </code></pre> |
| <p>If a different test suite has failed on CI, you will have to modify the <code>tests/ui</code> part.</p> |
| <p>To reproduce the whole CI job locally, you can run <code>cargo run --manifest-path src/ci/citool/Cargo.toml run-local x86_64-gnu-gcc</code>. |
| See <a href="tests/codegen-backend-tests/../docker.html">Testing with Docker</a> for more information.</p> |
| <h3 id="what-to-do-in-case-of-a-gcc-job-failure"><a class="header" href="#what-to-do-in-case-of-a-gcc-job-failure">What to do in case of a GCC job failure?</a></h3> |
| <p>If the GCC job test fails and it seems like the failure could be caused by the GCC backend, you can ping the <a href="https://github.com/orgs/rust-lang/teams/wg-gcc-backend">cg-gcc working group</a> using <code>@rust-lang/wg-gcc-backend</code></p> |
| <p>If fixing a compiler test that fails with the GCC backend is non-trivial, you can ignore that test when executed with <code>cg_gcc</code> using the <code>//@ ignore-backends: gcc</code> <a href="tests/codegen-backend-tests/../directives.html">compiletest directive</a>.</p> |
| <h2 id="choosing-which-codegen-backends-are-built"><a class="header" href="#choosing-which-codegen-backends-are-built">Choosing which codegen backends are built</a></h2> |
| <p>The <code>rust.codegen-backends = [...]</code> bootstrap option affects which codegen backends will be built and |
| included in the sysroot of the produced <code>rustc</code>. |
| To use the GCC codegen backend, <code>"gcc"</code> has to be included in this array in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">rust.codegen-backends = ["llvm", "gcc"] |
| </code></pre> |
| <p>If you don't want to change your <code>bootstrap.toml</code> file, you can alternatively run your <code>x</code> |
| commands with <code>--set 'rust.codegen-backends=["llvm", "gcc"]'</code>. |
| For example:</p> |
| <pre><code class="language-bash">./x build --set 'rust.codegen-backends=["llvm", "gcc"]' |
| </code></pre> |
| <p>The first backend in the <code>codegen-backends</code> array will determine which backend will be used as the |
| <em>default backend</em> of the built <code>rustc</code>. |
| This also determines which backend will be used to compile the |
| stage 1 standard library (or anything built in stage 2+). |
| To produce <code>rustc</code> that uses the GCC backend |
| by default, you can thus put <code>"gcc"</code> as the first element of this array:</p> |
| <pre><code class="language-bash">./x build --set 'rust.codegen-backends=["gcc"]' library |
| </code></pre> |
| <h2 id="choosing-the-codegen-backend-used-in-tests"><a class="header" href="#choosing-the-codegen-backend-used-in-tests">Choosing the codegen backend used in tests</a></h2> |
| <p>To run compiler tests with the GCC codegen backend being used to build the test Rust programs, you can use the |
| <code>--test-codegen-backend</code> flag:</p> |
| <pre><code class="language-bash">./x test tests/ui --test-codegen-backend gcc |
| </code></pre> |
| <p>Note that in order for this to work, the tested compiler must have the GCC codegen backend <a href="tests/codegen-backend-tests/cg_gcc.html#choosing-which-codegen-backends-are-built">available</a> in its sysroot directory.</p> |
| <h2 id="downloading-gcc-from-ci"><a class="header" href="#downloading-gcc-from-ci">Downloading GCC from CI</a></h2> |
| <p>The <code>gcc.download-ci-gcc</code> bootstrap option controls if GCC (which is a dependency of the GCC codegen backend) |
| will be downloaded from CI or built locally. |
| The default value is <code>true</code>, which will download GCC from CI |
| if there are no local changes to the GCC sources and the given host target is available on CI.</p> |
| <h2 id="running-tests-of-the-backend-itself"><a class="header" href="#running-tests-of-the-backend-itself">Running tests of the backend itself</a></h2> |
| <p>In addition to running the compiler's test suites using the GCC codegen backend, you can also run the test suite of the backend itself.</p> |
| <p>Now you do that using the following command:</p> |
| <pre><code class="language-text">./x test rustc_codegen_gcc |
| </code></pre> |
| <p>The backend needs to be <a href="tests/codegen-backend-tests/cg_gcc.html#choosing-which-codegen-backends-are-built">enabled</a> for this to work.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="performance-testing-1"><a class="header" href="#performance-testing-1">Performance testing</a></h1> |
| <h2 id="rustc-perf"><a class="header" href="#rustc-perf">rustc-perf</a></h2> |
| <p>A lot of work is put into improving the performance of the compiler and |
| preventing performance regressions.</p> |
| <p>The <a href="https://github.com/rust-lang/rustc-perf">rustc-perf</a> project provides |
| several services for testing and tracking performance. It provides hosted |
| infrastructure for running benchmarks as a service. At this time, only |
| <code>x86_64-unknown-linux-gnu</code> builds are tracked.</p> |
| <p>A "perf run" is used to compare the performance of the compiler in different |
| configurations for a large collection of popular crates. Different |
| configurations include "fresh builds", builds with incremental compilation, etc.</p> |
| <p>The result of a perf run is a comparison between two versions of the compiler |
| (by their commit hashes).</p> |
| <p>You can also use <code>rustc-perf</code> to manually benchmark and profile the compiler |
| <a href="tests/../profiling/with_rustc_perf.html">locally</a>.</p> |
| <h3 id="automatic-perf-runs"><a class="header" href="#automatic-perf-runs">Automatic perf runs</a></h3> |
| <p>After every PR is merged, a suite of benchmarks are run against the compiler. |
| The results are tracked over time on the <a href="https://perf.rust-lang.org/">https://perf.rust-lang.org/</a> website. |
| Any changes are noted in a comment on the PR.</p> |
| <h3 id="manual-perf-runs"><a class="header" href="#manual-perf-runs">Manual perf runs</a></h3> |
| <p>Additionally, performance tests can be ran before a PR is merged on an as-needed |
| basis. You should request a perf run if your PR may affect performance, |
| especially if it can affect performance adversely.</p> |
| <p>To evaluate the performance impact of a PR, write this comment on the PR:</p> |
| <p><code>@bors try @rust-timer queue</code></p> |
| <blockquote> |
| <p><strong>Note</strong>: Only users authorized to do perf runs are allowed to post this |
| comment. Teams that are allowed to use it are tracked in the <a href="https://github.com/rust-lang/team">Teams |
| repository</a> with the <code>perf = true</code> value in |
| the <code>[permissions]</code> section (and bors permissions are also required). If you |
| are not on one of those teams, feel free to ask for someone to post it for you |
| (either on Zulip or ask the assigned reviewer).</p> |
| </blockquote> |
| <p>This will first tell bors to do a "try" build which do a full release build for |
| <code>x86_64-unknown-linux-gnu</code>. After the build finishes, it will place it in the |
| queue to run the performance suite against it. After the performance tests |
| finish, the bot will post a comment on the PR with a summary and a link to a |
| full report.</p> |
| <p>If you want to do a perf run for an already built artifact (e.g. for a previous |
| try build that wasn't benchmarked yet), you can run this instead:</p> |
| <p><code>@rust-timer build <commit-sha></code></p> |
| <p>You cannot benchmark the same artifact twice though.</p> |
| <p>More information about the available perf bot commands can be found |
| <a href="https://perf.rust-lang.org/help.html">here</a>.</p> |
| <p>More details about the benchmarking process itself are available in the <a href="https://github.com/rust-lang/rustc-perf/blob/master/collector/README.md">perf |
| collector |
| documentation</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="miscellaneous-testing-related-info"><a class="header" href="#miscellaneous-testing-related-info">Miscellaneous testing-related info</a></h1> |
| <h2 id="rustc_bootstrap-and-stability"><a class="header" href="#rustc_bootstrap-and-stability"><code>RUSTC_BOOTSTRAP</code> and stability</a></h2> |
| <!-- date-check: Nov 2024 --> |
| <p>This is a bootstrap/compiler implementation detail, but it can also be useful |
| for testing:</p> |
| <ul> |
| <li><code>RUSTC_BOOTSTRAP=1</code> will "cheat" and bypass usual stability checking, allowing |
| you to use unstable features and cli flags on a stable <code>rustc</code>.</li> |
| <li><code>RUSTC_BOOTSTRAP=-1</code> will force a given <code>rustc</code> to pretend it is a stable |
| compiler, even if it's actually a nightly <code>rustc</code>. This is useful because some |
| behaviors of the compiler (e.g. diagnostics) can differ depending on whether |
| the compiler is nightly or not.</li> |
| </ul> |
| <p>In <code>ui</code> tests and other test suites that support <code>//@ rustc-env</code>, you can specify</p> |
| <pre><code class="language-rust ignore">// Force unstable features to be usable on stable rustc |
| //@ rustc-env:RUSTC_BOOTSTRAP=1 |
| |
| // Or force nightly rustc to pretend it is a stable rustc |
| //@ rustc-env:RUSTC_BOOTSTRAP=-1</code></pre> |
| <p>For <code>run-make</code>/<code>run-make-cargo</code> tests, <code>//@ rustc-env</code> is not supported. You can do |
| something like the following for individual <code>rustc</code> invocations.</p> |
| <pre><code class="language-rust ignore">use run_make_support::rustc; |
| |
| fn main() { |
| rustc() |
| // Pretend that I am very stable |
| .env("RUSTC_BOOTSTRAP", "-1") |
| //... |
| .run(); |
| }</code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="debugging-the-compiler"><a class="header" href="#debugging-the-compiler">Debugging the compiler</a></h1> |
| <p>This chapter contains a few tips to debug the compiler. These tips aim to be |
| useful no matter what you are working on. Some of the other chapters have |
| advice about specific parts of the compiler (e.g. the <a href="./incrcomp-debugging.html">Queries Debugging and |
| Testing chapter</a> or the <a href="./backend/debugging.html">LLVM Debugging |
| chapter</a>).</p> |
| <h2 id="configuring-the-compiler"><a class="header" href="#configuring-the-compiler">Configuring the compiler</a></h2> |
| <p>By default, rustc is built without most debug information. To enable debug info, |
| set <code>debug = true</code> in your bootstrap.toml.</p> |
| <p>Setting <code>debug = true</code> turns on many different debug options (e.g., <code>debug-assertions</code>, |
| <code>debug-logging</code>, etc.) which can be individually tweaked if you want to, but many people |
| simply set <code>debug = true</code>.</p> |
| <p>If you want to use GDB to debug rustc, please set <code>bootstrap.toml</code> with options:</p> |
| <pre><code class="language-toml">[rust] |
| debug = true |
| debuginfo-level = 2 |
| </code></pre> |
| <blockquote> |
| <p>NOTE: |
| This will use a lot of disk space |
| (upwards of <!-- date-check Aug 2022 --> 35GB), |
| and will take a lot more compile time. |
| With <code>debuginfo-level = 1</code> (the default when <code>debug = true</code>), |
| you will be able to track the execution path, |
| but will lose the symbol information for debugging.</p> |
| </blockquote> |
| <p>The default configuration will enable <code>symbol-mangling-version</code> v0. |
| This requires at least GDB v10.2, |
| otherwise you need to disable new symbol-mangling-version in <code>bootstrap.toml</code>.</p> |
| <pre><code class="language-toml">[rust] |
| new-symbol-mangling = false |
| </code></pre> |
| <blockquote> |
| <p>See the comments in <code>bootstrap.example.toml</code> for more info.</p> |
| </blockquote> |
| <p>You will need to rebuild the compiler after changing any configuration option.</p> |
| <h2 id="suppressing-the-ice-file"><a class="header" href="#suppressing-the-ice-file">Suppressing the ICE file</a></h2> |
| <p>By default, if rustc encounters an Internal Compiler Error (ICE) it will dump the ICE contents to an |
| ICE file within the current working directory named <code>rustc-ice-<timestamp>-<pid>.txt</code>. If this is |
| not desirable, you can prevent the ICE file from being created with <code>RUSTC_ICE=0</code>.</p> |
| <h2 id="getting-a-backtrace"><a class="header" href="#getting-a-backtrace">Getting a backtrace</a></h2> |
| <p>When you have an ICE (panic in the compiler), you can set |
| <code>RUST_BACKTRACE=1</code> to get the stack trace of the <code>panic!</code> like in |
| normal Rust programs. IIRC backtraces <strong>don't work</strong> on MinGW, |
| sorry. If you have trouble or the backtraces are full of <code>unknown</code>, |
| you might want to find some way to use Linux, Mac, or MSVC on Windows.</p> |
| <p>In the default configuration (without <code>debug</code> set to <code>true</code>), you don't have line numbers |
| enabled, so the backtrace looks like this:</p> |
| <pre><code class="language-text">stack backtrace: |
| 0: std::sys::imp::backtrace::tracing::imp::unwind_backtrace |
| 1: std::sys_common::backtrace::_print |
| 2: std::panicking::default_hook::{{closure}} |
| 3: std::panicking::default_hook |
| 4: std::panicking::rust_panic_with_hook |
| 5: std::panicking::begin_panic |
| (~~~~ LINES REMOVED BY ME FOR BREVITY ~~~~) |
| 32: rustc_typeck::check_crate |
| 33: <std::thread::local::LocalKey<T>>::with |
| 34: <std::thread::local::LocalKey<T>>::with |
| 35: rustc::ty::context::TyCtxt::create_and_enter |
| 36: rustc_driver::driver::compile_input |
| 37: rustc_driver::run_compiler |
| </code></pre> |
| <p>If you set <code>debug = true</code>, you will get line numbers for the stack trace. |
| Then the backtrace will look like this:</p> |
| <pre><code class="language-text">stack backtrace: |
| (~~~~ LINES REMOVED BY ME FOR BREVITY ~~~~) |
| at /home/user/rust/compiler/rustc_typeck/src/check/cast.rs:110 |
| 7: rustc_typeck::check::cast::CastCheck::check |
| at /home/user/rust/compiler/rustc_typeck/src/check/cast.rs:572 |
| at /home/user/rust/compiler/rustc_typeck/src/check/cast.rs:460 |
| at /home/user/rust/compiler/rustc_typeck/src/check/cast.rs:370 |
| (~~~~ LINES REMOVED BY ME FOR BREVITY ~~~~) |
| 33: rustc_driver::driver::compile_input |
| at /home/user/rust/compiler/rustc_driver/src/driver.rs:1010 |
| at /home/user/rust/compiler/rustc_driver/src/driver.rs:212 |
| 34: rustc_driver::run_compiler |
| at /home/user/rust/compiler/rustc_driver/src/lib.rs:253 |
| </code></pre> |
| <h2 id="-z-flags"><a class="header" href="#-z-flags"><code>-Z</code> flags</a></h2> |
| <p>The compiler has a bunch of <code>-Z *</code> flags. These are unstable flags that are only |
| enabled on nightly. Many of them are useful for debugging. To get a full listing |
| of <code>-Z</code> flags, use <code>-Z help</code>.</p> |
| <p>One useful flag is <code>-Z verbose-internals</code>, which generally enables printing more |
| info that could be useful for debugging.</p> |
| <p>Right below you can find elaborate explainers on a selected few.</p> |
| <h3 id="getting-a-backtrace-for-errors"><a class="header" href="#getting-a-backtrace-for-errors">Getting a backtrace for errors</a></h3> |
| <p>If you want to get a backtrace to the point where the compiler emits an |
| error message, you can pass the <code>-Z treat-err-as-bug=n</code>, which will make |
| the compiler panic on the <code>nth</code> error. If you leave off <code>=n</code>, the compiler will |
| assume <code>1</code> for <code>n</code> and thus panic on the first error it encounters.</p> |
| <p>For example:</p> |
| <pre><code class="language-bash">$ cat error.rs |
| </code></pre> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| 1 + (); |
| }</code></pre></pre> |
| <pre><code>$ rustc +stage1 error.rs |
| error[E0277]: cannot add `()` to `{integer}` |
| --> error.rs:2:7 |
| | |
| 2 | 1 + (); |
| | ^ no implementation for `{integer} + ()` |
| | |
| = help: the trait `Add<()>` is not implemented for `{integer}` |
| |
| error: aborting due to previous error |
| </code></pre> |
| <p>Now, where does the error above come from?</p> |
| <pre><code>$ RUST_BACKTRACE=1 rustc +stage1 error.rs -Z treat-err-as-bug |
| error[E0277]: the trait bound `{integer}: std::ops::Add<()>` is not satisfied |
| --> error.rs:2:7 |
| | |
| 2 | 1 + (); |
| | ^ no implementation for `{integer} + ()` |
| | |
| = help: the trait `std::ops::Add<()>` is not implemented for `{integer}` |
| |
| error: internal compiler error: unexpected panic |
| |
| note: the compiler unexpectedly panicked. this is a bug. |
| |
| note: we would appreciate a bug report: https://github.com/rust-lang/rust/blob/HEAD/CONTRIBUTING.md#bug-reports |
| |
| note: rustc 1.24.0-dev running on x86_64-unknown-linux-gnu |
| |
| note: run with `RUST_BACKTRACE=1` for a backtrace |
| |
| thread 'rustc' panicked at 'encountered error with `-Z treat_err_as_bug', |
| /home/user/rust/compiler/rustc_errors/src/lib.rs:411:12 |
| note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose |
| backtrace. |
| stack backtrace: |
| (~~~ IRRELEVANT PART OF BACKTRACE REMOVED BY ME ~~~) |
| 7: rustc::traits::error_reporting::<impl rustc::infer::InferCtxt<'a, 'tcx>> |
| ::report_selection_error |
| at /home/user/rust/compiler/rustc_middle/src/traits/error_reporting.rs:823 |
| 8: rustc::traits::error_reporting::<impl rustc::infer::InferCtxt<'a, 'tcx>> |
| ::report_fulfillment_errors |
| at /home/user/rust/compiler/rustc_middle/src/traits/error_reporting.rs:160 |
| at /home/user/rust/compiler/rustc_middle/src/traits/error_reporting.rs:112 |
| 9: rustc_typeck::check::FnCtxt::select_obligations_where_possible |
| at /home/user/rust/compiler/rustc_typeck/src/check/mod.rs:2192 |
| (~~~ IRRELEVANT PART OF BACKTRACE REMOVED BY ME ~~~) |
| 36: rustc_driver::run_compiler |
| at /home/user/rust/compiler/rustc_driver/src/lib.rs:253 |
| </code></pre> |
| <p>Cool, now I have a backtrace for the error!</p> |
| <h3 id="debugging-delayed-bugs"><a class="header" href="#debugging-delayed-bugs">Debugging delayed bugs</a></h3> |
| <p>The <code>-Z eagerly-emit-delayed-bugs</code> option makes it easy to debug delayed bugs. |
| It turns them into normal errors, i.e. makes them visible. This can be used in |
| combination with <code>-Z treat-err-as-bug</code> to stop at a particular delayed bug and |
| get a backtrace.</p> |
| <h3 id="getting-the-error-creation-location"><a class="header" href="#getting-the-error-creation-location">Getting the error creation location</a></h3> |
| <p><code>-Z track-diagnostics</code> can help figure out where errors are emitted. It uses <code>#[track_caller]</code> |
| for this and prints its location alongside the error:</p> |
| <pre><code>$ RUST_BACKTRACE=1 rustc +stage1 error.rs -Z track-diagnostics |
| error[E0277]: cannot add `()` to `{integer}` |
| --> src\error.rs:2:7 |
| | |
| 2 | 1 + (); |
| | ^ no implementation for `{integer} + ()` |
| -Ztrack-diagnostics: created at compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs:638:39 |
| | |
| = help: the trait `Add<()>` is not implemented for `{integer}` |
| = help: the following other types implement trait `Add<Rhs>`: |
| <&'a f32 as Add<f32>> |
| <&'a f64 as Add<f64>> |
| <&'a i128 as Add<i128>> |
| <&'a i16 as Add<i16>> |
| <&'a i32 as Add<i32>> |
| <&'a i64 as Add<i64>> |
| <&'a i8 as Add<i8>> |
| <&'a isize as Add<isize>> |
| and 48 others |
| |
| For more information about this error, try `rustc --explain E0277`. |
| </code></pre> |
| <p>This is similar but different to <code>-Z treat-err-as-bug</code>:</p> |
| <ul> |
| <li>it will print the locations for all errors emitted</li> |
| <li>it does not require a compiler built with debug symbols</li> |
| <li>you don't have to read through a big stack trace.</li> |
| </ul> |
| <h2 id="getting-logging-output"><a class="header" href="#getting-logging-output">Getting logging output</a></h2> |
| <p>The compiler uses the <a href="https://docs.rs/tracing"><code>tracing</code></a> crate for logging.</p> |
| <p>For details see <a href="./tracing.html">the guide section on tracing</a></p> |
| <h2 id="narrowing-bisecting-regressions"><a class="header" href="#narrowing-bisecting-regressions">Narrowing (Bisecting) Regressions</a></h2> |
| <p>The <a href="https://github.com/rust-lang/cargo-bisect-rustc">cargo-bisect-rustc</a> tool can be used as a quick and easy way to |
| find exactly which PR caused a change in <code>rustc</code> behavior. It automatically |
| downloads <code>rustc</code> PR artifacts and tests them against a project you provide |
| until it finds the regression. You can then look at the PR to get more context |
| on <em>why</em> it was changed. See <a href="https://rust-lang.github.io/cargo-bisect-rustc/tutorial.html">this tutorial</a> on how to use |
| it.</p> |
| <h2 id="downloading-artifacts-from-rusts-ci"><a class="header" href="#downloading-artifacts-from-rusts-ci">Downloading Artifacts from Rust's CI</a></h2> |
| <p>The <a href="https://github.com/kennytm/rustup-toolchain-install-master">rustup-toolchain-install-master</a> tool by kennytm can be used to |
| download the artifacts produced by Rust's CI for a specific SHA1 -- this |
| basically corresponds to the successful landing of some PR -- and then sets |
| them up for your local use. This also works for artifacts produced by <code>@bors try</code>. This is helpful when you want to examine the resulting build of a PR |
| without doing the build yourself.</p> |
| <h2 id="rustc_-test-attributes-1"><a class="header" href="#rustc_-test-attributes-1"><code>#[rustc_*]</code> TEST attributes</a></h2> |
| <p>The compiler defines a whole lot of internal (perma-unstable) attributes some of which are useful |
| for debugging by dumping extra compiler-internal information. These are prefixed with <code>rustc_</code> and |
| are gated behind the internal feature <code>rustc_attrs</code> (enabled via e.g. <code>#![feature(rustc_attrs)]</code>).</p> |
| <p>For a complete and up to date list, see <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_feature/src/builtin_attrs.rs"><code>builtin_attrs</code></a>. More specifically, the ones marked <code>TEST</code>. |
| Here are some notable ones:</p> |
| <div class="table-wrapper"><table><thead><tr><th>Attribute</th><th>Description</th></tr></thead><tbody> |
| <tr><td><code>rustc_def_path</code></td><td>Dumps the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.def_path_str"><code>def_path_str</code></a> of an item.</td></tr> |
| <tr><td><code>rustc_dump_def_parents</code></td><td>Dumps the chain of <code>DefId</code> parents of certain definitions.</td></tr> |
| <tr><td><code>rustc_dump_item_bounds</code></td><td>Dumps the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.item_bounds"><code>item_bounds</code></a> of an item.</td></tr> |
| <tr><td><code>rustc_dump_predicates</code></td><td>Dumps the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.predicates_of"><code>predicates_of</code></a> an item.</td></tr> |
| <tr><td><code>rustc_dump_vtable</code></td><td>Dumps the vtable layout of an impl, or a type alias of a dyn type.</td></tr> |
| <tr><td><code>rustc_hidden_type_of_opaques</code></td><td>Dumps the <a href="./opaque-types-impl-trait-inference.html">hidden type of each opaque types</a> in the crate.</td></tr> |
| <tr><td><code>rustc_layout</code></td><td><a href="compiler-debugging.html#debugging-type-layouts">See this section</a>.</td></tr> |
| <tr><td><code>rustc_object_lifetime_default</code></td><td>Dumps the <a href="https://doc.rust-lang.org/reference/lifetime-elision.html#default-trait-object-lifetimes">object lifetime defaults</a> of an item.</td></tr> |
| <tr><td><code>rustc_outlives</code></td><td>Dumps implied bounds of an item. More precisely, the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.inferred_outlives_of"><code>inferred_outlives_of</code></a> an item.</td></tr> |
| <tr><td><code>rustc_regions</code></td><td>Dumps NLL closure region requirements.</td></tr> |
| <tr><td><code>rustc_symbol_name</code></td><td>Dumps the mangled & demangled <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.symbol_name"><code>symbol_name</code></a> of an item.</td></tr> |
| <tr><td><code>rustc_variances</code></td><td>Dumps the <a href="./variance.html">variances</a> of an item.</td></tr> |
| </tbody></table> |
| </div> |
| <p>Right below you can find elaborate explainers on a selected few.</p> |
| <h3 id="formatting-graphviz-output-dot-files"><a class="header" href="#formatting-graphviz-output-dot-files">Formatting Graphviz output (.dot files)</a></h3> |
| <p>Some compiler options for debugging specific features yield graphviz graphs - |
| e.g. the <code>#[rustc_mir(borrowck_graphviz_postflow="suffix.dot")]</code> attribute |
| on a function dumps various borrow-checker dataflow graphs in conjunction with |
| <code>-Zdump-mir-dataflow</code>.</p> |
| <p>These all produce <code>.dot</code> files. To view these files, install graphviz (e.g. |
| <code>apt-get install graphviz</code>) and then run the following commands:</p> |
| <pre><code class="language-bash">$ dot -T pdf maybe_init_suffix.dot > maybe_init_suffix.pdf |
| $ firefox maybe_init_suffix.pdf # Or your favorite pdf viewer |
| </code></pre> |
| <h3 id="debugging-type-layouts"><a class="header" href="#debugging-type-layouts">Debugging type layouts</a></h3> |
| <p>The internal attribute <code>#[rustc_layout]</code> can be used to dump the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_public/abi/struct.Layout.html"><code>Layout</code></a> of |
| the type it is attached to. For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(rustc_attrs)] |
| |
| <span class="boring">fn main() { |
| </span>#[rustc_layout(debug)] |
| type T<'a> = &'a u32; |
| <span class="boring">}</span></code></pre></pre> |
| <p>Will emit the following:</p> |
| <pre><code class="language-text">error: layout_of(&'a u32) = Layout { |
| fields: Primitive, |
| variants: Single { |
| index: 0, |
| }, |
| abi: Scalar( |
| Scalar { |
| value: Pointer, |
| valid_range: 1..=18446744073709551615, |
| }, |
| ), |
| largest_niche: Some( |
| Niche { |
| offset: Size { |
| raw: 0, |
| }, |
| scalar: Scalar { |
| value: Pointer, |
| valid_range: 1..=18446744073709551615, |
| }, |
| }, |
| ), |
| align: AbiAndPrefAlign { |
| abi: Align { |
| pow2: 3, |
| }, |
| pref: Align { |
| pow2: 3, |
| }, |
| }, |
| size: Size { |
| raw: 8, |
| }, |
| } |
| --> src/lib.rs:4:1 |
| | |
| 4 | type T<'a> = &'a u32; |
| | ^^^^^^^^^^^^^^^^^^^^^ |
| |
| error: aborting due to previous error |
| </code></pre> |
| <h2 id="configuring-codelldb-for-debugging-rustc"><a class="header" href="#configuring-codelldb-for-debugging-rustc">Configuring CodeLLDB for debugging <code>rustc</code></a></h2> |
| <p>If you are using VSCode, and have edited your <code>bootstrap.toml</code> to request debugging |
| level 1 or 2 for the parts of the code you're interested in, then you should be |
| able to use the <a href="https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb">CodeLLDB</a> extension in VSCode to debug it.</p> |
| <p>Here is a sample <code>launch.json</code> file, being used to run a stage 1 compiler direct |
| from the directory where it is built (does not have to be "installed"):</p> |
| <pre><code class="language-javascript">// .vscode/launch.json |
| { |
| "version": "0.2.0", |
| "configurations": [ |
| { |
| "type": "lldb", |
| "request": "launch", |
| "name": "Launch", |
| "args": [], // array of string command-line arguments to pass to compiler |
| "program": "${workspaceFolder}/build/host/stage1/bin/rustc", |
| "windows": { // applicable if using windows |
| "program": "${workspaceFolder}/build/host/stage1/bin/rustc.exe" |
| }, |
| "cwd": "${workspaceFolder}", // current working directory at program start |
| "stopOnEntry": false, |
| "sourceLanguages": ["rust"] |
| } |
| ] |
| } |
| </code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="using-tracing-to-debug-the-compiler"><a class="header" href="#using-tracing-to-debug-the-compiler">Using tracing to debug the compiler</a></h1> |
| <p>The compiler has a lot of <a href="https://docs.rs/tracing/0.1/tracing/macro.debug.html"><code>debug!</code></a> (or <code>trace!</code>) calls, which print out logging information |
| at many points. These are very useful to at least narrow down the location of |
| a bug if not to find it entirely, or just to orient yourself as to why the |
| compiler is doing a particular thing.</p> |
| <p>To see the logs, you need to set the <code>RUSTC_LOG</code> environment variable to your |
| log filter. The full syntax of the log filters can be found in the <a href="https://docs.rs/tracing-subscriber/0.2.24/tracing_subscriber/filter/struct.EnvFilter.html#directives">rustdoc |
| of <code>tracing-subscriber</code></a>.</p> |
| <h2 id="function-level-filters"><a class="header" href="#function-level-filters">Function level filters</a></h2> |
| <p>Lots of functions in rustc are annotated with</p> |
| <pre><code>#[instrument(level = "debug", skip(self))] |
| fn foo(&self, bar: Type) {} |
| </code></pre> |
| <p>which allows you to use</p> |
| <pre><code>RUSTC_LOG=[foo] |
| </code></pre> |
| <p>to do the following all at once</p> |
| <ul> |
| <li>log all function calls to <code>foo</code></li> |
| <li>log the arguments (except for those in the <code>skip</code> list)</li> |
| <li>log everything (from anywhere else in the compiler) until the function returns</li> |
| </ul> |
| <h3 id="i-dont-want-everything"><a class="header" href="#i-dont-want-everything">I don't want everything</a></h3> |
| <p>Depending on the scope of the function, you may not want to log everything in its body. |
| As an example: the <code>do_mir_borrowck</code> function will dump hundreds of lines even for trivial |
| code being borrowchecked.</p> |
| <p>Since you can combine all filters, you can add a crate/module path, e.g.</p> |
| <pre><code>RUSTC_LOG=rustc_borrowck[do_mir_borrowck] |
| </code></pre> |
| <h3 id="i-dont-want-all-calls"><a class="header" href="#i-dont-want-all-calls">I don't want all calls</a></h3> |
| <p>If you are compiling libcore, you likely don't want <em>all</em> borrowck dumps, but only one |
| for a specific function. You can filter function calls by their arguments by regexing them.</p> |
| <pre><code>RUSTC_LOG=[do_mir_borrowck{id=\.\*from_utf8_unchecked\.\*}] |
| </code></pre> |
| <p>will only give you the logs of borrowchecking <code>from_utf8_unchecked</code>. Note that you will |
| still get a short message per ignored <code>do_mir_borrowck</code>, but none of the things inside those |
| calls. This helps you in looking through the calls that are happening and helps you adjust |
| your regex if you mistyped it.</p> |
| <h2 id="query-level-filters"><a class="header" href="#query-level-filters">Query level filters</a></h2> |
| <p>Every <a href="query.html">query</a> is automatically tagged with a logging span so that |
| you can display all log messages during the execution of the query. For |
| example, if you want to log everything during type checking:</p> |
| <pre><code>RUSTC_LOG=[typeck] |
| </code></pre> |
| <p>The query arguments are included as a tracing field which means that you can |
| filter on the debug display of the arguments. For example, the <code>typeck</code> query |
| has an argument <code>key: LocalDefId</code> of what is being checked. You can use a |
| regex to match on that <code>LocalDefId</code> to log type checking for a specific |
| function:</p> |
| <pre><code>RUSTC_LOG=[typeck{key=.*name_of_item.*}] |
| </code></pre> |
| <p>Different queries have different arguments. You can find a list of queries and |
| their arguments in |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_middle/src/query/mod.rs#L18"><code>rustc_middle/src/query/mod.rs</code></a>.</p> |
| <h2 id="broad-module-level-filters"><a class="header" href="#broad-module-level-filters">Broad module level filters</a></h2> |
| <p>You can also use filters similar to the <code>log</code> crate's filters, which will enable |
| everything within a specific module. This is often too verbose and too unstructured, |
| so it is recommended to use function level filters.</p> |
| <p>Your log filter can be just <code>debug</code> to get all <code>debug!</code> output and |
| higher (e.g., it will also include <code>info!</code>), or <code>path::to::module</code> to get <em>all</em> |
| output (which will include <code>trace!</code>) from a particular module, or |
| <code>path::to::module=debug</code> to get <code>debug!</code> output and higher from a particular |
| module.</p> |
| <p>For example, to get the <code>debug!</code> output and higher for a specific module, you |
| can run the compiler with <code>RUSTC_LOG=path::to::module=debug rustc my-file.rs</code>. |
| All <code>debug!</code> output will then appear in standard error.</p> |
| <p>Note that you can use a partial path and the filter will still work. For |
| example, if you want to see <code>info!</code> output from only |
| <code>rustdoc::passes::collect_intra_doc_links</code>, you could use |
| <code>RUSTDOC_LOG=rustdoc::passes::collect_intra_doc_links=info</code> <em>or</em> you could use |
| <code>RUSTDOC_LOG=rustdoc::passes::collect_intra=info</code>.</p> |
| <p>If you are developing rustdoc, use <code>RUSTDOC_LOG</code> instead. If you are developing |
| Miri, use <code>MIRI_LOG</code> instead. You get the idea :)</p> |
| <p>See the <a href="https://docs.rs/tracing"><code>tracing</code></a> crate's docs, and specifically the docs for <a href="https://docs.rs/tracing/0.1/tracing/macro.debug.html"><code>debug!</code></a> to |
| see the full syntax you can use. (Note: unlike the compiler, the <a href="https://docs.rs/tracing"><code>tracing</code></a> |
| crate and its examples use the <code>RUSTC_LOG</code> environment variable. rustc, rustdoc, |
| and other tools set custom environment variables.)</p> |
| <p><strong>Note that unless you use a very strict filter, the logger will emit a lot of |
| output, so use the most specific module(s) you can (comma-separated if |
| multiple)</strong>. It's typically a good idea to pipe standard error to a file and |
| look at the log output with a text editor.</p> |
| <p>So, to put it together:</p> |
| <pre><code class="language-bash"># This puts the output of all debug calls in `rustc_middle/src/traits` into |
| # standard error, which might fill your console backscroll. |
| $ RUSTC_LOG=rustc_middle::traits=debug rustc +stage1 my-file.rs |
| |
| # This puts the output of all debug calls in `rustc_middle/src/traits` in |
| # `traits-log`, so you can then see it with a text editor. |
| $ RUSTC_LOG=rustc_middle::traits=debug rustc +stage1 my-file.rs 2>traits-log |
| |
| # Not recommended! This will show the output of all `debug!` calls |
| # in the Rust compiler, and there are a *lot* of them, so it will be |
| # hard to find anything. |
| $ RUSTC_LOG=debug rustc +stage1 my-file.rs 2>all-log |
| |
| # This will show the output of all `info!` calls in `rustc_codegen_ssa`. |
| # |
| # There's an `info!` statement in `codegen_instance` that outputs |
| # every function that is codegen'd. This is useful to find out |
| # which function triggers an LLVM assertion, and this is an `info!` |
| # log rather than a `debug!` log so it will work on the official |
| # compilers. |
| $ RUSTC_LOG=rustc_codegen_ssa=info rustc +stage1 my-file.rs |
| |
| # This will show all logs in `rustc_codegen_ssa` and `rustc_resolve`. |
| $ RUSTC_LOG=rustc_codegen_ssa,rustc_resolve rustc +stage1 my-file.rs |
| |
| # This will show the output of all `info!` calls made by rustdoc |
| # or any rustc library it calls. |
| $ RUSTDOC_LOG=info rustdoc +stage1 my-file.rs |
| |
| # This will only show `debug!` calls made by rustdoc directly, |
| # not any `rustc*` crate. |
| $ RUSTDOC_LOG=rustdoc=debug rustdoc +stage1 my-file.rs |
| </code></pre> |
| <h2 id="log-colors"><a class="header" href="#log-colors">Log colors</a></h2> |
| <p>By default, rustc (and other tools, like rustdoc and Miri) will be smart about |
| when to use ANSI colors in the log output. If they are outputting to a terminal, |
| they will use colors, and if they are outputting to a file or being piped |
| somewhere else, they will not. However, it's hard to read log output in your |
| terminal unless you have a very strict filter, so you may want to pipe the |
| output to a pager like <code>less</code>. But then there won't be any colors, which makes |
| it hard to pick out what you're looking for!</p> |
| <p>You can override whether to have colors in log output with the <code>RUSTC_LOG_COLOR</code> |
| environment variable (or <code>RUSTDOC_LOG_COLOR</code> for rustdoc, or <code>MIRI_LOG_COLOR</code> |
| for Miri, etc.). There are three options: <code>auto</code> (the default), <code>always</code>, and |
| <code>never</code>. So, if you want to enable colors when piping to <code>less</code>, use something |
| similar to this command:</p> |
| <pre><code class="language-bash"># The `-R` switch tells less to print ANSI colors without escaping them. |
| $ RUSTC_LOG=debug RUSTC_LOG_COLOR=always rustc +stage1 ... | less -R |
| </code></pre> |
| <p>Note that <code>MIRI_LOG_COLOR</code> will only color logs that come from Miri, not logs |
| from rustc functions that Miri calls. Use <code>RUSTC_LOG_COLOR</code> to color logs from |
| rustc.</p> |
| <h2 id="how-to-keep-or-remove-debug-and-trace-calls-from-the-resulting-binary"><a class="header" href="#how-to-keep-or-remove-debug-and-trace-calls-from-the-resulting-binary">How to keep or remove <code>debug!</code> and <code>trace!</code> calls from the resulting binary</a></h2> |
| <p>While calls to <code>error!</code>, <code>warn!</code> and <code>info!</code> are included in every build of the compiler, |
| calls to <code>debug!</code> and <code>trace!</code> are only included in the program if |
| <code>debug-logging=true</code> is turned on in bootstrap.toml (it is |
| turned off by default), so if you don't see <code>DEBUG</code> logs, especially |
| if you run the compiler with <code>RUSTC_LOG=rustc rustc some.rs</code> and only see |
| <code>INFO</code> logs, make sure that <code>debug-logging=true</code> is turned on in your |
| bootstrap.toml.</p> |
| <h2 id="logging-etiquette-and-conventions"><a class="header" href="#logging-etiquette-and-conventions">Logging etiquette and conventions</a></h2> |
| <p>Because calls to <code>debug!</code> are removed by default, in most cases, don't worry |
| about the performance of adding "unnecessary" calls to <code>debug!</code> and leaving them in code you |
| commit - they won't slow down the performance of what we ship.</p> |
| <p>That said, there can also be excessive tracing calls, especially |
| when they are redundant with other calls nearby or in functions called from |
| here. There is no perfect balance to hit here, and is left to the reviewer's |
| discretion to decide whether to let you leave <code>debug!</code> statements in or whether to ask |
| you to remove them before merging.</p> |
| <p>It may be preferable to use <code>trace!</code> over <code>debug!</code> for very noisy logs.</p> |
| <p>A loosely followed convention is to use <code>#[instrument(level = "debug")]</code> |
| (<a href="https://docs.rs/tracing-attributes/0.1.17/tracing_attributes/attr.instrument.html">also see the attribute's documentation</a>) |
| in favour of <code>debug!("foo(...)")</code> at the start of a function <code>foo</code>. |
| Within functions, prefer <code>debug!(?variable.field)</code> over <code>debug!("xyz = {:?}", variable.field)</code> |
| and <code>debug!(bar = ?var.method(arg))</code> over <code>debug!("bar = {:?}", var.method(arg))</code>. |
| The documentation for this syntax can be found <a href="https://docs.rs/tracing/0.1.28/tracing/#recording-fields">here</a>.</p> |
| <p>One thing to be <strong>careful</strong> of is <strong>expensive</strong> operations in logs.</p> |
| <p>If in the module <code>rustc::foo</code> you have a statement</p> |
| <pre><code class="language-Rust">debug!(x = ?random_operation(tcx)); |
| </code></pre> |
| <p>Then if someone runs a debug <code>rustc</code> with <code>RUSTC_LOG=rustc::foo</code>, then |
| <code>random_operation()</code> will run. <code>RUSTC_LOG</code> filters that do not enable this |
| debug statement will not execute <code>random_operation</code>.</p> |
| <p>This means that you should not put anything too expensive or likely to crash |
| there - that would annoy anyone who wants to use logging for that module. |
| No-one will know it until someone tries to use logging to find <em>another</em> bug.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="profiling-the-compiler"><a class="header" href="#profiling-the-compiler">Profiling the compiler</a></h1> |
| <p>This section talks about how to profile the compiler and find out where it spends its time.</p> |
| <p>Depending on what you're trying to measure, there are several different approaches:</p> |
| <ul> |
| <li> |
| <p>If you want to see if a PR improves or regresses compiler performance, |
| see the <a href="tests/perf.html">rustc-perf chapter</a> for requesting a benchmarking run.</p> |
| </li> |
| <li> |
| <p>If you want a medium-to-high level overview of where <code>rustc</code> is spending its time:</p> |
| <ul> |
| <li>The <code>-Z self-profile</code> flag and <a href="https://github.com/rust-lang/measureme">measureme</a> tools offer a query-based approach to profiling. |
| See <a href="https://github.com/rust-lang/measureme/blob/master/summarize/README.md">their docs</a> for more information.</li> |
| </ul> |
| </li> |
| <li> |
| <p>If you want function level performance data or even just more details than the above approaches:</p> |
| <ul> |
| <li>Consider using a native code profiler such as <a href="profiling/with_perf.html">perf</a></li> |
| <li>or <a href="https://github.com/nagisa/rust_tracy_client">tracy</a> for a nanosecond-precision, |
| full-featured graphical interface.</li> |
| </ul> |
| </li> |
| <li> |
| <p>If you want a nice visual representation of the compile times of your crate graph, |
| you can use <a href="https://doc.rust-lang.org/nightly/cargo/reference/timings.html">cargo's <code>--timings</code> flag</a>, |
| e.g. <code>cargo build --timings</code>. |
| You can use this flag on the compiler itself with <code>CARGOFLAGS="--timings" ./x build</code></p> |
| </li> |
| <li> |
| <p>If you want to profile memory usage, you can use various tools depending on what operating system |
| you are using.</p> |
| <ul> |
| <li>For Windows, read our <a href="profiling/wpa_profiling.html">WPA guide</a>.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="optimizing-rustcs-bootstrap-times-with-cargo-llvm-lines"><a class="header" href="#optimizing-rustcs-bootstrap-times-with-cargo-llvm-lines">Optimizing rustc's bootstrap times with <code>cargo-llvm-lines</code></a></h2> |
| <p>Using <a href="https://github.com/dtolnay/cargo-llvm-lines">cargo-llvm-lines</a> you can count the |
| number of lines of LLVM IR across all instantiations of a generic function. |
| Since most of the time compiling rustc is spent in LLVM, the idea is that by |
| reducing the amount of code passed to LLVM, compiling rustc gets faster.</p> |
| <p>To use <code>cargo-llvm-lines</code> together with somewhat custom rustc build process, you can use |
| <code>-C save-temps</code> to obtain required LLVM IR. The option preserves temporary work products |
| created during compilation. Among those is LLVM IR that represents an input to the |
| optimization pipeline; ideal for our purposes. It is stored in files with <code>*.no-opt.bc</code> |
| extension in LLVM bitcode format.</p> |
| <p>Example usage:</p> |
| <pre><code>cargo install cargo-llvm-lines |
| # On a normal crate you could now run `cargo llvm-lines`, but `x` isn't normal :P |
| |
| # Do a clean before every run, to not mix in the results from previous runs. |
| ./x clean |
| env RUSTFLAGS=-Csave-temps ./x build --stage 0 compiler/rustc |
| |
| # Single crate, e.g., rustc_middle. (Relies on the glob support of your shell.) |
| # Convert unoptimized LLVM bitcode into a human readable LLVM assembly accepted by cargo-llvm-lines. |
| for f in build/x86_64-unknown-linux-gnu/stage0-rustc/x86_64-unknown-linux-gnu/release/deps/rustc_middle-*.no-opt.bc; do |
| ./build/x86_64-unknown-linux-gnu/llvm/bin/llvm-dis "$f" |
| done |
| cargo llvm-lines --files ./build/x86_64-unknown-linux-gnu/stage0-rustc/x86_64-unknown-linux-gnu/release/deps/rustc_middle-*.ll > llvm-lines-middle.txt |
| |
| # Specify all crates of the compiler. |
| for f in build/x86_64-unknown-linux-gnu/stage0-rustc/x86_64-unknown-linux-gnu/release/deps/*.no-opt.bc; do |
| ./build/x86_64-unknown-linux-gnu/llvm/bin/llvm-dis "$f" |
| done |
| cargo llvm-lines --files ./build/x86_64-unknown-linux-gnu/stage0-rustc/x86_64-unknown-linux-gnu/release/deps/*.ll > llvm-lines.txt |
| </code></pre> |
| <p>Example output for the compiler:</p> |
| <pre><code> Lines Copies Function name |
| ----- ------ ------------- |
| 45207720 (100%) 1583774 (100%) (TOTAL) |
| 2102350 (4.7%) 146650 (9.3%) core::ptr::drop_in_place |
| 615080 (1.4%) 8392 (0.5%) std::thread::local::LocalKey<T>::try_with |
| 594296 (1.3%) 1780 (0.1%) hashbrown::raw::RawTable<T>::rehash_in_place |
| 592071 (1.3%) 9691 (0.6%) core::option::Option<T>::map |
| 528172 (1.2%) 5741 (0.4%) core::alloc::layout::Layout::array |
| 466854 (1.0%) 8863 (0.6%) core::ptr::swap_nonoverlapping_one |
| 412736 (0.9%) 1780 (0.1%) hashbrown::raw::RawTable<T>::resize |
| 367776 (0.8%) 2554 (0.2%) alloc::raw_vec::RawVec<T,A>::grow_amortized |
| 367507 (0.8%) 643 (0.0%) rustc_query_system::dep_graph::graph::DepGraph<K>::with_task_impl |
| 355882 (0.8%) 6332 (0.4%) alloc::alloc::box_free |
| 354556 (0.8%) 14213 (0.9%) core::ptr::write |
| 354361 (0.8%) 3590 (0.2%) core::iter::traits::iterator::Iterator::fold |
| 347761 (0.8%) 3873 (0.2%) rustc_middle::ty::context::tls::set_tlv |
| 337534 (0.7%) 2377 (0.2%) alloc::raw_vec::RawVec<T,A>::allocate_in |
| 331690 (0.7%) 3192 (0.2%) hashbrown::raw::RawTable<T>::find |
| 328756 (0.7%) 3978 (0.3%) rustc_middle::ty::context::tls::with_context_opt |
| 326903 (0.7%) 642 (0.0%) rustc_query_system::query::plumbing::try_execute_query |
| </code></pre> |
| <p>Since this doesn't seem to work with incremental compilation or <code>./x check</code>, |
| you will be compiling rustc <em>a lot</em>. |
| I recommend changing a few settings in <code>bootstrap.toml</code> to make it bearable:</p> |
| <pre><code>[rust] |
| # A debug build takes _a third_ as long on my machine, |
| # but compiling more than stage0 rustc becomes unbearably slow. |
| optimize = false |
| |
| # We can't use incremental anyway, so we disable it for a little speed boost. |
| incremental = false |
| # We won't be running it, so no point in compiling debug checks. |
| debug = false |
| |
| # Using a single codegen unit gives less output, but is slower to compile. |
| codegen-units = 0 # num_cpus |
| </code></pre> |
| <p>The llvm-lines output is affected by several options. |
| <code>optimize = false</code> increases it from 2.1GB to 3.5GB and <code>codegen-units = 0</code> to 4.1GB.</p> |
| <p>MIR optimizations have little impact. Compared to the default <code>RUSTFLAGS="-Z mir-opt-level=1"</code>, level 0 adds 0.3GB and level 2 removes 0.2GB. |
| As of <!-- date-check --> July 2022, |
| inlining happens in LLVM and GCC codegen backends, |
| missing only in the Cranelift one.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="profiling-with-perf"><a class="header" href="#profiling-with-perf">Profiling with perf</a></h1> |
| <p>This is a guide for how to profile rustc with <a href="https://perf.wiki.kernel.org/index.php/Main_Page">perf</a>.</p> |
| <h2 id="initial-steps"><a class="header" href="#initial-steps">Initial steps</a></h2> |
| <ul> |
| <li>Get a clean checkout of rust-lang/rust</li> |
| <li>Set the following settings in your <code>bootstrap.toml</code>: |
| <ul> |
| <li><code>rust.debuginfo-level = 1</code> - enables line debuginfo</li> |
| <li><code>rust.jemalloc = false</code> - lets you do memory use profiling with valgrind</li> |
| <li>leave everything else the defaults</li> |
| </ul> |
| </li> |
| <li>Run <code>./x build</code> to get a full build</li> |
| <li>Make a rustup toolchain pointing to that result |
| <ul> |
| <li>see <a href="profiling/../building/how-to-build-and-run.html#toolchain">the "build and run" section for instructions</a></li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="gathering-a-perf-profile"><a class="header" href="#gathering-a-perf-profile">Gathering a perf profile</a></h2> |
| <p>perf is an excellent tool on linux that can be used to gather and |
| analyze all kinds of information. Mostly it is used to figure out |
| where a program spends its time. It can also be used for other sorts |
| of events, though, like cache misses and so forth.</p> |
| <h3 id="the-basics"><a class="header" href="#the-basics">The basics</a></h3> |
| <p>The basic <code>perf</code> command is this:</p> |
| <pre><code class="language-bash">perf record -F99 --call-graph dwarf XXX |
| </code></pre> |
| <p>The <code>-F99</code> tells perf to sample at 99 Hz, which avoids generating too |
| much data for longer runs (why 99 Hz you ask? It is often chosen |
| because it is unlikely to be in lockstep with other periodic |
| activity). The <code>--call-graph dwarf</code> tells perf to get call-graph |
| information from debuginfo, which is accurate. The <code>XXX</code> is the |
| command you want to profile. So, for example, you might do:</p> |
| <pre><code class="language-bash">perf record -F99 --call-graph dwarf cargo +<toolchain> rustc |
| </code></pre> |
| <p>to run <code>cargo</code> -- here <code><toolchain></code> should be the name of the toolchain |
| you made in the beginning. But there are some things to be aware of:</p> |
| <ul> |
| <li>You probably don't want to profile the time spend building |
| dependencies. So something like <code>cargo build; cargo clean -p $C</code> may |
| be helpful (where <code>$C</code> is the crate name) |
| <ul> |
| <li>Though usually I just do <code>touch src/lib.rs</code> and rebuild instead. =)</li> |
| </ul> |
| </li> |
| <li>You probably don't want incremental messing about with your |
| profile. So something like <code>CARGO_INCREMENTAL=0</code> can be helpful.</li> |
| </ul> |
| <p>In case to avoid the issue of <code>addr2line xxx/elf: could not read first record</code> when reading |
| collected data from <code>cargo</code>, you may need use the latest version of <code>addr2line</code>:</p> |
| <pre><code class="language-bash">cargo install addr2line --features="bin" |
| </code></pre> |
| <h3 id="gathering-a-perf-profile-from-a-perfrust-langorg-test"><a class="header" href="#gathering-a-perf-profile-from-a-perfrust-langorg-test">Gathering a perf profile from a <code>perf.rust-lang.org</code> test</a></h3> |
| <p>Often we want to analyze a specific test from <code>perf.rust-lang.org</code>. |
| The easiest way to do that is to use the <a href="https://github.com/rust-lang/rustc-perf">rustc-perf</a> |
| benchmarking suite, this approach is described <a href="profiling/with_rustc_perf.html">here</a>.</p> |
| <p>Instead of using the benchmark suite CLI, you can also profile the benchmarks manually. First, |
| you need to clone the <a href="https://github.com/rust-lang/rustc-perf">rustc-perf</a> repository:</p> |
| <pre><code class="language-bash">$ git clone https://github.com/rust-lang/rustc-perf |
| </code></pre> |
| <p>and then find the source code of the test that you want to profile. Sources for the tests |
| are found in <a href="https://github.com/rust-lang/rustc-perf/tree/master/collector/compile-benchmarks">the <code>collector/compile-benchmarks</code> directory</a> |
| and <a href="https://github.com/rust-lang/rustc-perf/tree/master/collector/runtime-benchmarks">the <code>collector/runtime-benchmarks</code> directory</a>. So let's |
| go into the directory of a specific test; we'll use <code>clap-rs</code> as an example:</p> |
| <pre><code class="language-bash">cd collector/compile-benchmarks/clap-3.1.6 |
| </code></pre> |
| <p>In this case, let's say we want to profile the <code>cargo check</code> |
| performance. In that case, I would first run some basic commands to |
| build the dependencies:</p> |
| <pre><code class="language-bash"># Setup: first clean out any old results and build the dependencies: |
| cargo +<toolchain> clean |
| CARGO_INCREMENTAL=0 cargo +<toolchain> check |
| </code></pre> |
| <p>(Again, <code><toolchain></code> should be replaced with the name of the |
| toolchain we made in the first step.)</p> |
| <p>Next: we want record the execution time for <em>just</em> the clap-rs crate, |
| running cargo check. I tend to use <code>cargo rustc</code> for this, since it |
| also allows me to add explicit flags, which we'll do later on.</p> |
| <pre><code class="language-bash">touch src/lib.rs |
| CARGO_INCREMENTAL=0 perf record -F99 --call-graph dwarf cargo rustc --profile check --lib |
| </code></pre> |
| <p>Note that final command: it's a doozy! It uses the <code>cargo rustc</code> |
| command, which executes rustc with (potentially) additional options; |
| the <code>--profile check</code> and <code>--lib</code> options specify that we are doing a |
| <code>cargo check</code> execution, and that this is a library (not a binary).</p> |
| <p>At this point, we can use <code>perf</code> tooling to analyze the results. For example:</p> |
| <pre><code class="language-bash">perf report |
| </code></pre> |
| <p>will open up an interactive TUI program. In simple cases, that can be |
| helpful. For more detailed examination, the <a href="https://github.com/nikomatsakis/perf-focus"><code>perf-focus</code> tool</a> |
| can be helpful; it is covered below.</p> |
| <p><strong>A note of caution.</strong> Each of the rustc-perf tests is its own special |
| snowflake. In particular, some of them are not libraries, in which |
| case you would want to do <code>touch src/main.rs</code> and avoid passing |
| <code>--lib</code>. I'm not sure how best to tell which test is which to be |
| honest.</p> |
| <h3 id="gathering-nll-data"><a class="header" href="#gathering-nll-data">Gathering NLL data</a></h3> |
| <p>If you want to profile an NLL run, you can just pass extra options to |
| the <code>cargo rustc</code> command, like so:</p> |
| <pre><code class="language-bash">touch src/lib.rs |
| CARGO_INCREMENTAL=0 perf record -F99 --call-graph dwarf cargo rustc --profile check --lib -- -Z borrowck=mir |
| </code></pre> |
| <h2 id="analyzing-a-perf-profile-with-perf-focus"><a class="header" href="#analyzing-a-perf-profile-with-perf-focus">Analyzing a perf profile with <code>perf focus</code></a></h2> |
| <p>Once you've gathered a perf profile, we want to get some information |
| about it. For this, I personally use <a href="https://github.com/nikomatsakis/perf-focus">perf focus</a>. It's a kind of |
| simple but useful tool that lets you answer queries like:</p> |
| <ul> |
| <li>"how much time was spent in function F" (no matter where it was called from)</li> |
| <li>"how much time was spent in function F when it was called from G"</li> |
| <li>"how much time was spent in function F <em>excluding</em> time spent in G"</li> |
| <li>"what functions does F call and how much time does it spend in them"</li> |
| </ul> |
| <p>To understand how it works, you have to know just a bit about |
| perf. Basically, perf works by <em>sampling</em> your process on a regular |
| basis (or whenever some event occurs). For each sample, perf gathers a |
| backtrace. <code>perf focus</code> lets you write a regular expression that tests |
| which functions appear in that backtrace, and then tells you which |
| percentage of samples had a backtrace that met the regular |
| expression. It's probably easiest to explain by walking through how I |
| would analyze NLL performance.</p> |
| <h3 id="installing-perf-focus"><a class="header" href="#installing-perf-focus">Installing <code>perf-focus</code></a></h3> |
| <p>You can install perf-focus using <code>cargo install</code>:</p> |
| <pre><code class="language-bash">cargo install perf-focus |
| </code></pre> |
| <h3 id="example-how-much-time-is-spent-in-mir-borrowck"><a class="header" href="#example-how-much-time-is-spent-in-mir-borrowck">Example: How much time is spent in MIR borrowck?</a></h3> |
| <p>Let's say we've gathered the NLL data for a test. We'd like to know |
| how much time it is spending in the MIR borrow-checker. The "main" |
| function of the MIR borrowck is called <code>do_mir_borrowck</code>, so we can do |
| this command:</p> |
| <pre><code class="language-bash">$ perf focus '{do_mir_borrowck}' |
| Matcher : {do_mir_borrowck} |
| Matches : 228 |
| Not Matches: 542 |
| Percentage : 29% |
| </code></pre> |
| <p>The <code>'{do_mir_borrowck}'</code> argument is called the <strong>matcher</strong>. It |
| specifies the test to be applied on the backtrace. In this case, the |
| <code>{X}</code> indicates that there must be <em>some</em> function on the backtrace |
| that meets the regular expression <code>X</code>. In this case, that regex is |
| just the name of the function we want (in fact, it's a subset of the name; |
| the full name includes a bunch of other stuff, like the module |
| path). In this mode, perf-focus just prints out the percentage of |
| samples where <code>do_mir_borrowck</code> was on the stack: in this case, 29%.</p> |
| <p><strong>A note about c++filt.</strong> To get the data from <code>perf</code>, <code>perf focus</code> |
| currently executes <code>perf script</code> (perhaps there is a better |
| way...). I've sometimes found that <code>perf script</code> outputs C++ mangled |
| names. This is annoying. You can tell by running <code>perf script | head</code> yourself — if you see names like <code>5rustc6middle</code> instead of |
| <code>rustc::middle</code>, then you have the same problem. You can solve this |
| by doing:</p> |
| <pre><code class="language-bash">perf script | c++filt | perf focus --from-stdin ... |
| </code></pre> |
| <p>This will pipe the output from <code>perf script</code> through <code>c++filt</code> and |
| should mostly convert those names into a more friendly format. The |
| <code>--from-stdin</code> flag to <code>perf focus</code> tells it to get its data from |
| stdin, rather than executing <code>perf focus</code>. We should make this more |
| convenient (at worst, maybe add a <code>c++filt</code> option to <code>perf focus</code>, or |
| just always use it — it's pretty harmless).</p> |
| <h3 id="example-how-much-time-does-mir-borrowck-spend-solving-traits"><a class="header" href="#example-how-much-time-does-mir-borrowck-spend-solving-traits">Example: How much time does MIR borrowck spend solving traits?</a></h3> |
| <p>Perhaps we'd like to know how much time MIR borrowck spends in the |
| trait checker. We can ask this using a more complex regex:</p> |
| <pre><code class="language-bash">$ perf focus '{do_mir_borrowck}..{^rustc::traits}' |
| Matcher : {do_mir_borrowck},..{^rustc::traits} |
| Matches : 12 |
| Not Matches: 1311 |
| Percentage : 0% |
| </code></pre> |
| <p>Here we used the <code>..</code> operator to ask "how often do we have |
| <code>do_mir_borrowck</code> on the stack and then, later, some function whose |
| name begins with <code>rustc::traits</code>?" (basically, code in that module). It |
| turns out the answer is "almost never" — only 12 samples fit that |
| description (if you ever see <em>no</em> samples, that often indicates your |
| query is messed up).</p> |
| <p>If you're curious, you can find out exactly which samples by using the |
| <code>--print-match</code> option. This will print out the full backtrace for |
| each sample. The <code>|</code> at the front of the line indicates the part that |
| the regular expression matched.</p> |
| <h3 id="example-where-does-mir-borrowck-spend-its-time"><a class="header" href="#example-where-does-mir-borrowck-spend-its-time">Example: Where does MIR borrowck spend its time?</a></h3> |
| <p>Often we want to do more "explorational" queries. Like, we know that |
| MIR borrowck is 29% of the time, but where does that time get spent? |
| For that, the <code>--tree-callees</code> option is often the best tool. You |
| usually also want to give <code>--tree-min-percent</code> or |
| <code>--tree-max-depth</code>. The result looks like this:</p> |
| <pre><code class="language-bash">$ perf focus '{do_mir_borrowck}' --tree-callees --tree-min-percent 3 |
| Matcher : {do_mir_borrowck} |
| Matches : 577 |
| Not Matches: 746 |
| Percentage : 43% |
| |
| Tree |
| | matched `{do_mir_borrowck}` (43% total, 0% self) |
| : | rustc_borrowck::nll::compute_regions (20% total, 0% self) |
| : : | rustc_borrowck::nll::type_check::type_check_internal (13% total, 0% self) |
| : : : | core::ops::function::FnOnce::call_once (5% total, 0% self) |
| : : : : | rustc_borrowck::nll::type_check::liveness::generate (5% total, 3% self) |
| : : : | <rustc_borrowck::nll::type_check::TypeVerifier<'a, 'b, 'tcx> as rustc::mir::visit::Visitor<'tcx>>::visit_mir (3% total, 0% self) |
| : | rustc::mir::visit::Visitor::visit_mir (8% total, 6% self) |
| : | <rustc_borrowck::MirBorrowckCtxt<'cx, 'tcx> as rustc_mir_dataflow::DataflowResultsConsumer<'cx, 'tcx>>::visit_statement_entry (5% total, 0% self) |
| : | rustc_mir_dataflow::do_dataflow (3% total, 0% self) |
| </code></pre> |
| <p>What happens with <code>--tree-callees</code> is that</p> |
| <ul> |
| <li>we find each sample matching the regular expression</li> |
| <li>we look at the code that occurs <em>after</em> the regex match and try |
| to build up a call tree</li> |
| </ul> |
| <p>The <code>--tree-min-percent 3</code> option says "only show me things that take |
| more than 3% of the time". Without this, the tree often gets really |
| noisy and includes random stuff like the innards of |
| malloc. <code>--tree-max-depth</code> can be useful too, it just limits how many |
| levels we print.</p> |
| <p>For each line, we display the percent of time in that function |
| altogether ("total") and the percent of time spent in <strong>just that |
| function and not some callee of that function</strong> (self). Usually |
| "total" is the more interesting number, but not always.</p> |
| <h3 id="relative-percentages"><a class="header" href="#relative-percentages">Relative percentages</a></h3> |
| <p>By default, all in perf-focus are relative to the <strong>total program |
| execution</strong>. This is useful to help you keep perspective — often as |
| we drill down to find hot spots, we can lose sight of the fact that, |
| in terms of overall program execution, this "hot spot" is actually not |
| important. It also ensures that percentages between different queries |
| are easily compared against one another.</p> |
| <p>That said, sometimes it's useful to get relative percentages, so <code>perf focus</code> offers a <code>--relative</code> option. In this case, the percentages are |
| listed only for samples that match (vs all samples). So for example we |
| could get our percentages relative to the borrowck itself |
| like so:</p> |
| <pre><code class="language-bash">$ perf focus '{do_mir_borrowck}' --tree-callees --relative --tree-max-depth 1 --tree-min-percent 5 |
| Matcher : {do_mir_borrowck} |
| Matches : 577 |
| Not Matches: 746 |
| Percentage : 100% |
| |
| Tree |
| | matched `{do_mir_borrowck}` (100% total, 0% self) |
| : | rustc_borrowck::nll::compute_regions (47% total, 0% self) [...] |
| : | rustc::mir::visit::Visitor::visit_mir (19% total, 15% self) [...] |
| : | <rustc_borrowck::MirBorrowckCtxt<'cx, 'tcx> as rustc_mir_dataflow::DataflowResultsConsumer<'cx, 'tcx>>::visit_statement_entry (13% total, 0% self) [...] |
| : | rustc_mir_dataflow::do_dataflow (8% total, 1% self) [...] |
| </code></pre> |
| <p>Here you see that <code>compute_regions</code> came up as "47% total" — that |
| means that 47% of <code>do_mir_borrowck</code> is spent in that function. Before, |
| we saw 20% — that's because <code>do_mir_borrowck</code> itself is only 43% of |
| the total time (and <code>.47 * .43 = .20</code>).</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="profiling-on-windows"><a class="header" href="#profiling-on-windows">Profiling on Windows</a></h1> |
| <h2 id="introducing-wpr-and-wpa"><a class="header" href="#introducing-wpr-and-wpa">Introducing WPR and WPA</a></h2> |
| <p>High-level performance analysis (including memory usage) can be performed with the Windows |
| Performance Recorder (WPR) and Windows Performance Analyzer (WPA). As the names suggest, WPR is for |
| recording system statistics (in the form of event trace log a.k.a. ETL files), while WPA is for |
| analyzing these ETL files.</p> |
| <p>WPR collects system wide statistics, so it won't just record things relevant to rustc but also |
| everything else that's running on the machine. During analysis, we can filter to just the things we |
| find interesting.</p> |
| <p>These tools are quite powerful but also require a bit of learning |
| before we can successfully profile the Rust compiler.</p> |
| <p>Here we will explore how to use WPR and WPA for analyzing the Rust compiler as well as provide |
| links to useful "profiles" (i.e., settings files that tweak the defaults for WPR and WPA) that are |
| specifically designed to make analyzing rustc easier.</p> |
| <h3 id="installing-wpr-and-wpa"><a class="header" href="#installing-wpr-and-wpa">Installing WPR and WPA</a></h3> |
| <p>You can install WPR and WPA as part of the Windows Performance Toolkit which itself is an option as |
| part of downloading the Windows Assessment and Deployment Kit (ADK). You can download the ADK |
| installer <a href="https://go.microsoft.com/fwlink/?linkid=2086042">here</a>. Make sure to select the Windows |
| Performance Toolkit (you don't need to select anything else).</p> |
| <h2 id="recording"><a class="header" href="#recording">Recording</a></h2> |
| <p>In order to perform system analysis, you'll first need to record your system with WPR. Open WPR and |
| at the bottom of the window select the "profiles" of the things you want to record. For looking |
| into memory usage of the rustc bootstrap process, we'll want to select the following items:</p> |
| <ul> |
| <li>CPU usage</li> |
| <li>VirtualAlloc usage</li> |
| </ul> |
| <p>You might be tempted to record "Heap usage" as well, but this records every single heap allocation |
| and can be very, very expensive. For high-level analysis, it might be best to leave that turned |
| off.</p> |
| <p>Now we need to get our setup ready to record. For memory usage analysis, it is best to record the |
| stage 2 compiler build with a stage 1 compiler build with debug symbols. Having symbols in the |
| compiler we're using to build rustc will aid our analysis greatly by allowing WPA to resolve Rust |
| symbols correctly. Unfortunately, the stage 0 compiler does not have symbols turned on which is why |
| we'll need to build a stage 1 compiler and then a stage 2 compiler ourselves.</p> |
| <p>To do this, make sure you have set <code>debuginfo-level = 1</code> in your <code>bootstrap.toml</code> file. This tells |
| rustc to generate debug information which includes stack frames when bootstrapping.</p> |
| <p>Now you can build the stage 1 compiler: <code>x build --stage 1 -i library</code> or however |
| else you want to build the stage 1 compiler.</p> |
| <p>Now that the stage 1 compiler is built, we can record the stage 2 build. Go back to WPR, click the |
| "start" button and build the stage 2 compiler (e.g., <code>x build --stage=2 -i library</code>). |
| When this process finishes, stop the recording.</p> |
| <p>Click the Save button and once that process is complete, click the "Open in WPA" button which |
| appears.</p> |
| <blockquote> |
| <p>Note: The trace file is fairly large so it can take WPA some time to finish opening the file.</p> |
| </blockquote> |
| <h2 id="analysis"><a class="header" href="#analysis">Analysis</a></h2> |
| <p>Now that our ETL file is open in WPA, we can analyze the results. First, we'll want to apply the |
| pre-made "profile" which will put WPA into a state conducive to analyzing rustc bootstrap. Download |
| the profile <a href="https://github.com/wesleywiser/rustc-bootstrap-wpa-analysis/releases/download/1/rustc.generic.wpaProfile">here</a>. |
| Select the "Profiles" menu at the top, then "apply" and then choose the downloaded profile.</p> |
| <p>You should see something resembling the following:</p> |
| <p><img src="profiling/../img/wpa-initial-memory.png" alt="WPA with profile applied" /></p> |
| <p>Next, we will need to tell WPA to load and process debug symbols so that it can properly demangle |
| the Rust stack traces. To do this, click "Trace" and then choose "Load Symbols". This step can take |
| a while.</p> |
| <p>Once WPA has loaded symbols for rustc, we can expand the rustc.exe node and begin drilling down |
| into the stack with the largest allocations.</p> |
| <p>To do that, we'll expand the <code>[Root]</code> node in the "Commit Stack" column and continue expanding |
| until we find interesting stack frames.</p> |
| <blockquote> |
| <p>Tip: After selecting the node you want to expand, press the right arrow key. This will expand the |
| node and put the selection on the next largest node in the expanded set. You can continue pressing |
| the right arrow key until you reach an interesting frame.</p> |
| </blockquote> |
| <p><img src="profiling/../img/wpa-stack.png" alt="WPA with expanded stack" /></p> |
| <p>In this sample, you can see calls through codegen are allocating ~30gb of memory in total |
| throughout this profile.</p> |
| <h2 id="other-analysis-tabs"><a class="header" href="#other-analysis-tabs">Other Analysis Tabs</a></h2> |
| <p>The profile also includes a few other tabs which can be helpful:</p> |
| <ul> |
| <li>System Configuration |
| <ul> |
| <li>General information about the system the capture was recorded on.</li> |
| </ul> |
| </li> |
| <li>rustc Build Processes |
| <ul> |
| <li>A flat list of relevant processes such as rustc.exe, cargo.exe, link.exe etc.</li> |
| <li>Each process lists its command line arguments.</li> |
| <li>Useful for figuring out what a specific rustc process was working on.</li> |
| </ul> |
| </li> |
| <li>rustc Build Process Tree |
| <ul> |
| <li>Timeline showing when processes started and exited.</li> |
| </ul> |
| </li> |
| <li>rustc CPU Analysis |
| <ul> |
| <li>Contains charts preconfigured to show hotspots in rustc.</li> |
| <li>These charts are designed to support analyzing where rustc is spending its time.</li> |
| </ul> |
| </li> |
| <li>rustc Memory Analysis |
| <ul> |
| <li>Contains charts preconfigured to show where rustc is allocating memory.</li> |
| </ul> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="profiling-with-rustc-perf"><a class="header" href="#profiling-with-rustc-perf">Profiling with rustc-perf</a></h1> |
| <p>The <a href="https://github.com/rust-lang/rustc-perf">Rust benchmark suite</a> provides a comprehensive way of profiling and benchmarking |
| the Rust compiler. You can find instructions on how to use the suite in its <a href="https://github.com/rust-lang/rustc-perf/blob/master/collector/README.md">manual</a>.</p> |
| <p>However, using the suite manually can be a bit cumbersome. To make this easier for <code>rustc</code> contributors, |
| the compiler build system (<code>bootstrap</code>) also provides built-in integration with the benchmarking suite, |
| which will download and build the suite for you, build a local compiler toolchain and let you profile it using a simplified command-line interface.</p> |
| <p>You can use the <code>./x perf <command> [options]</code> command to use this integration.</p> |
| <p>You can use normal bootstrap flags for this command, such as <code>--stage 1</code> or <code>--stage 2</code>, for example to modify the stage of the created sysroot. It might also be useful to configure <code>bootstrap.toml</code> to better support profiling, e.g. set <code>rust.debuginfo-level = 1</code> to add source line information to the built compiler.</p> |
| <p><code>x perf</code> currently supports the following commands:</p> |
| <ul> |
| <li><code>benchmark <id></code>: Benchmark the compiler and store the results under the passed <code>id</code>.</li> |
| <li><code>compare <baseline> <modified></code>: Compare the benchmark results of two compilers with the two passed <code>id</code>s.</li> |
| <li><code>eprintln</code>: Just run the compiler and capture its <code>stderr</code> output. Note that the compiler normally does not print |
| anything to <code>stderr</code>, you might want to add some <code>eprintln!</code> calls to get any output.</li> |
| <li><code>samply</code>: Profile the compiler using the <a href="https://github.com/mstange/samply">samply</a> sampling profiler.</li> |
| <li><code>cachegrind</code>: Use <a href="https://www.cs.cmu.edu/afs/cs.cmu.edu/project/cmt-40/Nice/RuleRefinement/bin/valgrind-3.2.0/docs/html/cg-manual.html">Cachegrind</a> to generate a detailed simulated trace of the compiler's execution.</li> |
| </ul> |
| <blockquote> |
| <p>You can find a more detailed description of the profilers in the <a href="https://github.com/rust-lang/rustc-perf/blob/master/collector/README.md#profiling-local-builds"><code>rustc-perf</code> manual</a>.</p> |
| </blockquote> |
| <p>You can use the following options for the <code>x perf</code> command, which mirror the corresponding options of the |
| <code>profile_local</code> and <code>bench_local</code> commands that you can use in the suite:</p> |
| <ul> |
| <li><code>--include</code>: Select benchmarks which should be profiled/benchmarked.</li> |
| <li><code>--profiles</code>: Select profiles (<code>Check</code>, <code>Debug</code>, <code>Opt</code>, <code>Doc</code>) which should be profiled/benchmarked.</li> |
| <li><code>--scenarios</code>: Select scenarios (<code>Full</code>, <code>IncrFull</code>, <code>IncrPatched</code>, <code>IncrUnchanged</code>) which should be profiled/benchmarked.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="cratesio-dependencies"><a class="header" href="#cratesio-dependencies">crates.io dependencies</a></h1> |
| <p>The Rust compiler supports building with some dependencies from <code>crates.io</code>. |
| Examples are <code>log</code> and <code>env_logger</code>.</p> |
| <p>In general, |
| you should avoid adding dependencies to the compiler for several reasons:</p> |
| <ul> |
| <li>The dependency may not be of high quality or well-maintained.</li> |
| <li>The dependency may not be using a compatible license.</li> |
| <li>The dependency may have transitive dependencies that have one of the above |
| problems.</li> |
| </ul> |
| <!-- date-check: Aug 2025 --> |
| <p>Note that there is no official policy for vetting new dependencies to the compiler. |
| Decisions are made on a case-by-case basis, during code review.</p> |
| <h2 id="permitted-dependencies"><a class="header" href="#permitted-dependencies">Permitted dependencies</a></h2> |
| <p>The <code>tidy</code> tool has <a href="https://github.com/rust-lang/rust/blob/9d1b2106e23b1abd32fce1f17267604a5102f57a/src/tools/tidy/src/deps.rs#L73">a list of crates that are allowed</a>. To add a |
| dependency that is not already in the compiler, you will need to add it to the list.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="contribution-procedures"><a class="header" href="#contribution-procedures">Contribution procedures</a></h1> |
| <h2 id="bug-reports"><a class="header" href="#bug-reports">Bug reports</a></h2> |
| <p>While bugs are unfortunate, they're a reality in software. |
| We can't fix what we don't know about, so please report liberally. |
| If you're not sure if something is a bug, feel free to open an issue anyway.</p> |
| <p><strong>If you believe reporting your bug publicly represents a security risk to Rust users, |
| please follow our <a href="https://www.rust-lang.org/policies/security">instructions for reporting security vulnerabilities</a></strong>.</p> |
| <p>If you're using the nightly channel, please check if the bug exists in the |
| latest toolchain before filing your bug. |
| It might be fixed already.</p> |
| <p>If you have the chance, before reporting a bug, please <a href="https://github.com/rust-lang/rust/issues?q=is%3Aissue">search existing issues</a>, |
| as it's possible that someone else has already reported your error. |
| This doesn't always work, and sometimes it's hard to know what to search for, so consider this |
| extra credit. |
| We won't mind if you accidentally file a duplicate report.</p> |
| <p>Similarly, to help others who encountered the bug find your issue, consider |
| filing an issue with a descriptive title, which contains information that might be unique to it. |
| This can be the language or compiler feature used, the |
| conditions that trigger the bug, or part of the error message if there is any. |
| An example could be: <strong>"impossible case reached" on lifetime inference for impl |
| Trait in return position</strong>.</p> |
| <p>Opening an issue is as easy as following <a href="https://github.com/rust-lang/rust/issues/new/choose">this link</a> and filling out the fields |
| in the appropriate provided template.</p> |
| <h2 id="bug-fixes-or-normal-code-changes"><a class="header" href="#bug-fixes-or-normal-code-changes">Bug fixes or "normal" code changes</a></h2> |
| <p>For most PRs, no special procedures are needed. |
| You can just <a href="contributing.html#pull-requests">open a PR</a>, and it will be reviewed, approved, and merged. |
| This includes most bug fixes, refactorings, and other user-invisible changes. |
| The next few sections talk about exceptions to this rule.</p> |
| <p>Also, note that it is perfectly acceptable to open WIP PRs or GitHub <a href="https://github.blog/2019-02-14-introducing-draft-pull-requests/">Draft PRs</a>. |
| Some people prefer to do this so they can get feedback along the |
| way or share their code with a collaborator. |
| Others do this so they can utilize |
| the CI to build and test their PR (e.g. when developing on a slow machine).</p> |
| <h2 id="new-features"><a class="header" href="#new-features">New features</a></h2> |
| <p>Rust has strong backwards-compatibility guarantees. |
| Thus, new features can't just be implemented directly in stable Rust. |
| Instead, we have 3 release channels: stable, beta, and nightly. |
| See <a href="https://doc.rust-lang.org/book/appendix-07-nightly-rust.html">The Rust Book</a> for more details on Rust’s train release model.</p> |
| <ul> |
| <li><strong>Stable</strong>: this is the latest stable release for general usage.</li> |
| <li><strong>Beta</strong>: this is the next release (will be stable within 6 weeks).</li> |
| <li><strong>Nightly</strong>: follows the <code>main</code> branch of the repo. |
| This is the only channel where unstable features are intended to be used, |
| which happens via opt-in feature gates.</li> |
| </ul> |
| <p>See <a href="./implementing_new_features.html">this chapter on implementing new features</a> for more |
| information.</p> |
| <h3 id="breaking-changes"><a class="header" href="#breaking-changes">Breaking changes</a></h3> |
| <p>Breaking changes have a <a href="bug-fix-procedure.html">dedicated section</a> in the dev-guide.</p> |
| <h3 id="major-changes"><a class="header" href="#major-changes">Major changes</a></h3> |
| <p>The compiler team has a special process for large changes, whether or not they cause breakage. |
| This process is called a Major Change Proposal (MCP). |
| MCP is a relatively lightweight mechanism for getting feedback on large changes to the |
| compiler (as opposed to a full RFC or a design meeting with the team).</p> |
| <p>Example of things that might require MCPs include major refactorings, changes |
| to important types, or important changes to how the compiler does something, or |
| smaller user-facing changes.</p> |
| <p><strong>When in doubt, ask <a href="https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler">on Zulip</a>. |
| It would be a shame to put a lot of work |
| into a PR that ends up not getting merged!</strong> <a href="https://forge.rust-lang.org/compiler/proposals-and-stabilization.html#how-do-i-submit-an-mcp">See this document</a> for more info on MCPs.</p> |
| <h3 id="performance"><a class="header" href="#performance">Performance</a></h3> |
| <p>Compiler performance is important. |
| We have put a lot of effort over the last few years into <a href="https://perf.rust-lang.org/dashboard.html">gradually improving it</a>.</p> |
| <p>If you suspect that your change may cause a performance regression (or |
| improvement), you can request a "perf run" (and your reviewer may also request one |
| before approving). |
| This is yet another bot that will compile a collection of |
| benchmarks on a compiler with your changes. |
| The numbers are reported |
| <a href="https://perf.rust-lang.org">here</a>, and you can see a comparison of your changes against the latest <code>main</code>.</p> |
| <blockquote> |
| <p>For an introduction to the performance of Rust code in general |
| which would also be useful in rustc development, see <a href="https://nnethercote.github.io/perf-book/">The Rust Performance Book</a>.</p> |
| </blockquote> |
| <h2 id="pull-requests"><a class="header" href="#pull-requests">Pull requests</a></h2> |
| <p>Pull requests (or PRs for short) are the primary mechanism we use to change Rust. |
| GitHub itself has some <a href="https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests">great documentation</a> on using the Pull Request feature. |
| We use the <a href="https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/getting-started/about-collaborative-development-models#fork-and-pull-model">"fork and pull" model</a>, |
| where contributors push changes to their personal fork and create pull requests to |
| bring those changes into the source repository. |
| We have <a href="git.html">a chapter</a> on how to use Git when contributing to Rust.</p> |
| <blockquote> |
| <p><strong>Advice for potentially large, complex, cross-cutting and/or very domain-specific changes</strong></p> |
| <p>The compiler reviewers on rotation usually each have areas of the compiler that they know well, |
| but also have areas that they are not very familiar with. If your PR contains changes that are |
| large, complex, cross-cutting and/or highly domain-specific, it becomes very difficult to find a |
| suitable reviewer who is comfortable in reviewing all of the changes in such a PR. This is also |
| true if the changes are not only compiler-specific but also contains changes which fall under the |
| purview of reviewers from other teams, like the standard library team. <a href="https://github.com/rust-lang/rust/blob/HEAD/triagebot.toml">There's a bot</a> |
| which notifies the relevant teams and pings people who have setup specific alerts based on the |
| files modified.</p> |
| <p>Before making such changes, you are strongly encouraged to <strong>discuss your proposed changes with |
| the compiler team beforehand</strong> (and with other teams that the changes would require approval |
| from), and work with the compiler team to see if we can help you <strong>break down a large potentially |
| unreviewable PR into a series of smaller more individually reviewable PRs</strong>.</p> |
| <p>You can communicate with the compiler team by creating a <a href="https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler">#t-compiler thread on Zulip</a> |
| to discuss your proposed changes.</p> |
| <p>Communicating with the compiler team beforehand helps in several ways:</p> |
| <ol> |
| <li>It increases the likelihood of your PRs being reviewed in a timely manner. |
| <ul> |
| <li>We can help you identify suitable reviewers <em>before</em> you open actual PRs, or help find |
| advisors and liaisons to help you navigate the change procedures, or help with running |
| try-jobs, perf runs and crater runs as suitable.</li> |
| </ul> |
| </li> |
| <li>It helps the compiler team track your changes.</li> |
| <li>The compiler team can perform vibe checks on your changes early and often, to see if the |
| direction of the changes align with what the compiler team prefers to see.</li> |
| <li>Helps to avoid situations where you may have invested significant time and effort into large |
| changes that the compiler team might not be willing to accept, or finding out very late that the |
| changes are in a direction that the compiler team disagrees with.</li> |
| </ol> |
| </blockquote> |
| <h3 id="keeping-your-branch-up-to-date"><a class="header" href="#keeping-your-branch-up-to-date">Keeping your branch up-to-date</a></h3> |
| <p>The CI in rust-lang/rust applies your patches directly against current <code>main</code>, |
| not against the commit your branch is based on. |
| This can lead to unexpected failures |
| if your branch is outdated, even when there are no explicit merge conflicts.</p> |
| <p>Update your branch only when needed: when you have merge conflicts, upstream CI is broken and blocking your green PR, or a maintainer requests it. |
| Avoid updating an already-green PR under review unless necessary. |
| During review, make incremental commits to address feedback. |
| Prefer to squash or rebase only at the end, or when a reviewer requests it.</p> |
| <p>When updating, use <code>git push --force-with-lease</code> and leave a brief comment explaining what changed. |
| Some repos prefer merging from <code>upstream/main</code> instead of rebasing; |
| follow the project's conventions. |
| See <a href="git.html#keeping-things-up-to-date">keeping things up to date</a> for detailed instructions.</p> |
| <p>After rebasing, it's recommended to <a href="tests/intro.html">run the relevant tests locally</a> to catch any issues before CI runs.</p> |
| <h3 id="r"><a class="header" href="#r">r?</a></h3> |
| <p>All pull requests are reviewed by another person. |
| We have a bot, <a href="https://github.com/rustbot">@rustbot</a>, that will automatically assign a random person |
| to review your request based on which files you changed.</p> |
| <p>If you want to request that a specific person reviews your pull request, you |
| can add an <code>r?</code> to the pull request description or in a comment. |
| For example, if you want to ask a review by @awesome-reviewer, |
| add the following to the end of the pull request description:</p> |
| <pre><code>r? @awesome-reviewer |
| </code></pre> |
| <p><a href="https://github.com/rustbot">@rustbot</a> will then assign the PR to that reviewer instead of a random person. |
| This is entirely optional.</p> |
| <p>You can also assign a random reviewer from a specific team by writing <code>r? rust-lang/groupname</code>. |
| As an example, if you were making a diagnostics change, |
| you could get a reviewer from the diagnostics team by adding:</p> |
| <pre><code>r? rust-lang/diagnostics |
| </code></pre> |
| <p>For a full list of possible <code>groupname</code>s, |
| check the <code>adhoc_groups</code> section at the <a href="https://github.com/rust-lang/rust/blob/HEAD/triagebot.toml">triagebot.toml config file</a>, |
| or the list of teams in the <a href="https://github.com/rust-lang/team/tree/HEAD/teams">rust-lang teams database</a>.</p> |
| <h3 id="waiting-for-reviews"><a class="header" href="#waiting-for-reviews">Waiting for reviews</a></h3> |
| <blockquote> |
| <p>NOTE</p> |
| <p>Pull request reviewers are often working at capacity, |
| and many of them are contributing on a volunteer basis. |
| In order to minimize review delays, |
| pull request authors and assigned reviewers should ensure that the review label |
| (<code>S-waiting-on-review</code> and <code>S-waiting-on-author</code>) stays updated, |
| invoking these commands when appropriate:</p> |
| <ul> |
| <li> |
| <p><code>@rustbot author</code>: |
| the review is finished, |
| and PR author should check the comments and take action accordingly.</p> |
| </li> |
| <li> |
| <p><code>@rustbot review</code>: |
| the author is ready for a review, |
| and this PR will be queued again in the reviewer's queue.</p> |
| </li> |
| </ul> |
| </blockquote> |
| <p>Please note that the reviewers are humans, who for the most part work on <code>rustc</code> in their free time. |
| This means that they can take some time to respond and review your PR. |
| It also means that reviewers can miss some PRs that are assigned to them.</p> |
| <p>To try to move PRs forward, the Triage WG regularly goes through all PRs that |
| are waiting for review and haven't been discussed for at least 2 weeks. |
| If you don't get a review within 2 weeks, feel free to ask the Triage WG on |
| Zulip (<a href="https://rust-lang.zulipchat.com/#narrow/stream/242269-t-release.2Ftriage">#t-release/triage</a>). |
| They have knowledge of when to ping, who might be on vacation, etc.</p> |
| <p>The reviewer may request some changes using the GitHub code review interface. |
| They may also request special procedures for some PRs. |
| See <a href="tests/crater.html">Crater</a> and <a href="bug-fix-procedure.html">Breaking Changes</a> chapters for some examples of such procedures.</p> |
| <h3 id="ci"><a class="header" href="#ci">CI</a></h3> |
| <p>In addition to being reviewed by a human, pull requests are automatically tested, |
| thanks to continuous integration (CI). |
| Basically, every time you open and update |
| a pull request, CI builds the compiler and tests it against the |
| <a href="tests/intro.html">compiler test suite</a>, and also performs other tests such as checking that |
| your pull request is in compliance with Rust's style guidelines.</p> |
| <p>Running continuous integration tests allows PR authors to catch mistakes early |
| without going through a first review cycle, and also helps reviewers stay aware |
| of the status of a particular pull request.</p> |
| <p>Rust has plenty of CI capacity, and you should never have to worry about wasting |
| computational resources each time you push a change. |
| It is also perfectly fine |
| (and even encouraged!) to use the CI to test your changes if it can help your productivity. |
| In particular, we don't recommend running the full <code>./x test</code> suite locally, |
| since it takes a very long time to execute.</p> |
| <h3 id="r-1"><a class="header" href="#r-1">r+</a></h3> |
| <p>After someone has reviewed your pull request, they will leave an annotation |
| on the pull request with an <code>r+</code>. |
| It will look something like this:</p> |
| <pre><code>@bors r+ |
| </code></pre> |
| <p>This tells <a href="https://github.com/bors">@bors</a>, our lovable integration bot, that your pull request has been approved. |
| The PR then enters the <a href="https://bors.rust-lang.org/queue/rust">merge queue</a>, where <a href="https://github.com/bors">@bors</a> |
| will run <em>all</em> the tests on <em>every</em> platform we support. |
| If it all works out, <a href="https://github.com/bors">@bors</a> will merge your code into <code>main</code> and close the pull request.</p> |
| <p>Depending on the scale of the change, you may see a slightly different form of <code>r+</code>:</p> |
| <pre><code>@bors r+ rollup |
| </code></pre> |
| <p>The additional <code>rollup</code> tells <a href="https://github.com/bors">@bors</a> that this change should always be "rolled up". |
| Changes that are rolled up are tested and merged alongside other PRs, to speed the process up. |
| Typically, only small changes that are expected not to conflict |
| with one another are marked as "always roll up".</p> |
| <p>Be patient; |
| this can take a while and the queue can sometimes be long. |
| Also, note that PRs are never merged by hand.</p> |
| <h3 id="opening-a-pr"><a class="header" href="#opening-a-pr">Opening a PR</a></h3> |
| <p>You are now ready to file a pull request (PR)? |
| Great! |
| Here are a few points you should be aware of.</p> |
| <p>All pull requests should be filed against the <code>main</code> branch, |
| unless you know for sure that you should target a different branch.</p> |
| <p>Run some style checks before you submit the PR:</p> |
| <pre><code>./x test tidy --bless |
| </code></pre> |
| <p>We recommend to make this check before every pull request (and every new commit in a pull request); |
| you can add <a href="https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks">git hooks</a> before every push to make sure you never forget to make this check. |
| The CI will also run tidy and will fail if tidy fails.</p> |
| <p>Rust follows a <em>no merge-commit policy</em>, |
| meaning that when you encounter merge conflicts, |
| you are expected to always rebase instead of merging. |
| For example, |
| always use rebase when bringing the latest changes from the <code>main</code> branch to your feature branch. |
| If your PR contains merge commits, it will get marked as <code>has-merge-commits</code>. |
| Once you have removed the merge commits, e.g., through an interactive rebase, you |
| should remove the label again:</p> |
| <pre><code>@rustbot label -has-merge-commits |
| </code></pre> |
| <p>See <a href="./rustbot.html#issue-relabeling">this chapter</a> for more details.</p> |
| <p>If you encounter merge conflicts or when a reviewer asks you to perform some |
| changes, your PR will get marked as <code>S-waiting-on-author</code>. |
| When you resolve them, you should use <code>@rustbot</code> to mark it as <code>S-waiting-on-review</code>:</p> |
| <pre><code>@rustbot ready |
| </code></pre> |
| <p>GitHub allows <a href="https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue">closing issues using keywords</a>. |
| This feature should be used to keep the issue tracker tidy. |
| However, it is generally preferred |
| to put the "closes #123" text in the PR description rather than the issue commit; |
| particularly during rebasing, citing the issue number in the commit can "spam" |
| the issue in question.</p> |
| <p>However, if your PR fixes a stable-to-beta or stable-to-stable regression and has |
| been accepted for a beta and/or stable backport (i.e., it is marked <code>beta-accepted</code> |
| and/or <code>stable-accepted</code>), please do <em>not</em> use any such keywords since we don't |
| want the corresponding issue to get auto-closed once the fix lands on <code>main</code>. |
| Please update the PR description while still mentioning the issue somewhere. |
| For example, you could write <code>Fixes (after beta backport) #NNN.</code>.</p> |
| <p>As for further actions, please keep a sharp look-out for a PR whose title begins with |
| <code>[beta]</code> or <code>[stable]</code> and which backports the PR in question. |
| When that one gets merged, the relevant issue can be closed. |
| The closing comment should mention all PRs that were involved. |
| If you don't have the permissions to close the issue, please |
| leave a comment on the original PR asking the reviewer to close it for you.</p> |
| <h3 id="reverting-a-pr"><a class="header" href="#reverting-a-pr">Reverting a PR</a></h3> |
| <p>When a PR leads to miscompile, significant performance regressions, or other critical issues, we may |
| want to revert that PR with a regression test case. |
| You can also check out the <a href="https://forge.rust-lang.org/compiler/reviews.html?highlight=revert#reverts">revert policy</a> on |
| Forge docs (which is mainly targeted for reviewers, but contains useful info for PR authors too).</p> |
| <p>If the PR contains huge changes, it can be challenging to revert, making it harder to review |
| incremental fixes in subsequent updates. |
| Or if certain code in that PR is heavily depended upon by |
| subsequent PRs, reverting it can become difficult.</p> |
| <p>In such cases, we can identify the problematic code and disable it for some input, as shown in <a href="https://github.com/rust-lang/rust/pull/128271">#128271</a>.</p> |
| <p>For MIR optimizations, we can also use the <code>-Zunsound-mir-opt</code> option to gate the mir-opt, as shown |
| in <a href="https://github.com/rust-lang/rust/pull/132356">#132356</a>.</p> |
| <h2 id="external-dependencies"><a class="header" href="#external-dependencies">External dependencies</a></h2> |
| <p>This section has moved to <a href="./external-repos.html">"Using External Repositories"</a>.</p> |
| <h2 id="writing-documentation"><a class="header" href="#writing-documentation">Writing documentation</a></h2> |
| <p>Documentation improvements are very welcome. |
| The source of <code>doc.rust-lang.org</code> |
| is located in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/doc"><code>src/doc</code></a> in the tree, and standard API documentation is generated |
| from the source code itself (e.g. <a href="https://github.com/rust-lang/rust/blob/HEAD/library/std/src/lib.rs#L1"><code>library/std/src/lib.rs</code></a>). Documentation pull requests |
| function in the same way as other pull requests.</p> |
| <p>To find documentation-related issues, use the <a href="https://github.com/rust-lang/rust/issues?q=is%3Aopen%20is%3Aissue%20label%3AA-docs">A-docs label</a>.</p> |
| <p>You can find documentation style guidelines in <a href="https://github.com/rust-lang/rfcs/blob/master/text/1574-more-api-documentation-conventions.md#appendix-a-full-conventions-text">RFC 1574</a>.</p> |
| <p>To build the standard library documentation, use <code>x doc --stage 1 library --open</code>. |
| To build the documentation for a book (e.g. the unstable book), use <code>x doc src/doc/unstable-book.</code> |
| Results should appear in <code>build/host/doc</code>, as well as automatically open in your default browser. |
| See <a href="./building/compiler-documenting.html#building-documentation">Building Documentation</a> for more |
| information.</p> |
| <p>You can also use <code>rustdoc</code> directly to check small fixes. |
| For example, <code>rustdoc src/doc/reference.md</code> will render reference to <code>doc/reference.html</code>. |
| The CSS might be messed up, but you can verify that the HTML is right.</p> |
| <p>Please notice that we don't accept typography/spellcheck fixes to <strong>internal documentation</strong> |
| as it's usually not worth the churn or the review time. |
| Examples of internal documentation is code comments and rustc api docs. |
| However, feel free to fix those if accompanied by other improvements in the same PR.</p> |
| <h3 id="contributing-to-rustc-dev-guide"><a class="header" href="#contributing-to-rustc-dev-guide">Contributing to rustc-dev-guide</a></h3> |
| <p>Contributions to the <a href="https://rustc-dev-guide.rust-lang.org/">rustc-dev-guide</a> are always welcome, and can be made directly at |
| <a href="https://github.com/rust-lang/rustc-dev-guide">the rust-lang/rustc-dev-guide repo</a>. |
| The issue tracker in that repo is also a great way to find things that need doing. |
| There are issues for beginners and advanced compiler devs alike!</p> |
| <p>Just a few things to keep in mind:</p> |
| <ul> |
| <li> |
| <p>Please try to avoid overly long lines and use semantic line breaks (where you break the line after each sentence). |
| There is no strict limit on line lengths; |
| let the sentence or part of the sentence flow to its proper end on the same line.</p> |
| <p>You can use a tool in ci/sembr to help with this. |
| Its help output can be seen with this command:</p> |
| <pre><code class="language-console">cargo run --manifest-path ci/sembr/Cargo.toml -- --help |
| </code></pre> |
| </li> |
| <li> |
| <p>When contributing text to the guide, please contextualize the information with some time period |
| and/or a reason so that the reader knows how much to trust the information. |
| Aim to provide a reasonable amount of context, possibly including but not limited to:</p> |
| <ul> |
| <li> |
| <p>A reason for why the text may be out of date other than "change", |
| as change is a constant across the project.</p> |
| </li> |
| <li> |
| <p>The date the comment was added, e.g. instead of writing <em>"Currently, ..."</em> |
| or <em>"As of now, ..."</em>, consider adding the date, in one of the following formats:</p> |
| <ul> |
| <li>Jan 2021</li> |
| <li>January 2021</li> |
| <li>jan 2021</li> |
| <li>january 2021</li> |
| </ul> |
| <p>There is a CI action (in <code>.github/workflows/date-check.yml</code>) |
| that generates a monthly report showing those that are over 6 months old |
| (<a href="https://github.com/rust-lang/rustc-dev-guide/issues/2052">example</a>).</p> |
| <p>For the action to pick the date, add a special annotation before specifying the date:</p> |
| <pre><code class="language-md"><!-- date-check --> Nov 2025 |
| </code></pre> |
| <p>Example:</p> |
| <pre><code class="language-md">As of <!-- date-check --> Nov 2025, the foo did the bar. |
| </code></pre> |
| <p>For cases where the date should not be part of the visible rendered output, |
| use the following instead:</p> |
| <pre><code class="language-md"><!-- date-check: Nov 2025 --> |
| </code></pre> |
| </li> |
| <li> |
| <p>A link to a relevant WG, tracking issue, <code>rustc</code> rustdoc page, or similar, that may provide |
| further explanation for the change process or a way to verify that the information is not |
| outdated.</p> |
| </li> |
| </ul> |
| </li> |
| <li> |
| <p>If a text grows rather long (more than a few page scrolls) or complicated (more than four |
| subsections), it might benefit from having a Table of Contents at the beginning, |
| which you can auto-generate by including the <code><!-- toc --></code> marker at the top.</p> |
| </li> |
| </ul> |
| <h4 id="-note-where-to-contribute-rustc-dev-guide-changes"><a class="header" href="#-note-where-to-contribute-rustc-dev-guide-changes">⚠️ Note: Where to contribute <code>rustc-dev-guide</code> changes</a></h4> |
| <p>For detailed information about where to contribute rustc-dev-guide changes and the benefits of doing so, |
| see <a href="https://forge.rust-lang.org/wg-rustc-dev-guide/index.html#where-to-contribute-rustc-dev-guide-changes">the rustc-dev-guide working group documentation</a>.</p> |
| <h2 id="issue-triage"><a class="header" href="#issue-triage">Issue triage</a></h2> |
| <p>Please see <a href="https://forge.rust-lang.org/release/issue-triaging.html">https://forge.rust-lang.org/release/issue-triaging.html</a>.</p> |
| <h3 id="rfcbot-labels"><a class="header" href="#rfcbot-labels">rfcbot labels</a></h3> |
| <p><a href="https://github.com/anp/rfcbot-rs/">rfcbot</a> uses its own labels for tracking the process of coordinating |
| asynchronous decisions, such as approving or rejecting a change. |
| This is used for <a href="https://github.com/rust-lang/rfcs">RFCs</a>, issues, and pull requests.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Labels</th><th>Color</th><th>Description</th></tr></thead><tbody> |
| <tr><td><a href="https://github.com/rust-lang/rust/labels/proposed-final-comment-period">proposed-final-comment-period</a></td><td><span class="label-color" style="background-color:#ededed;"> </span> Gray</td><td>Currently awaiting signoff of all team members in order to enter the final comment period.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rust/labels/disposition-merge">disposition-merge</a></td><td><span class="label-color" style="background-color:#008800;"> </span> Green</td><td>Indicates the intent is to merge the change.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rust/labels/disposition-close">disposition-close</a></td><td><span class="label-color" style="background-color:#dd0000;"> </span> Red</td><td>Indicates the intent is to not accept the change and close it.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rust/labels/disposition-postpone">disposition-postpone</a></td><td><span class="label-color" style="background-color:#ededed;"> </span> Gray</td><td>Indicates the intent is to not accept the change at this time and postpone it to a later date.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rust/labels/final-comment-period">final-comment-period</a></td><td><span class="label-color" style="background-color:#1e76d9;"> </span> Blue</td><td>Currently soliciting final comments before merging or closing.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rust/labels/finished-final-comment-period">finished-final-comment-period</a></td><td><span class="label-color" style="background-color:#f9e189;"> </span> Light Yellow</td><td>The final comment period has concluded, and the issue will be merged or closed.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rfcs/labels/postponed">postponed</a></td><td><span class="label-color" style="background-color:#fbca04;"> </span> Yellow</td><td>The issue has been postponed.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rfcs/labels/closed">closed</a></td><td><span class="label-color" style="background-color:#dd0000;"> </span> Red</td><td>The issue has been rejected.</td></tr> |
| <tr><td><a href="https://github.com/rust-lang/rfcs/labels/to-announce">to-announce</a></td><td><span class="label-color" style="background-color:#ededed;"> </span> Gray</td><td>Issues that have finished their final-comment-period and should be publicly announced. Note: the rust-lang/rust repository uses this label differently, to announce issues at the triage meetings.</td></tr> |
| </tbody></table> |
| </div> |
| <h2 id="helpful-links-and-information"><a class="header" href="#helpful-links-and-information">Helpful links and information</a></h2> |
| <p>This section has moved to the <a href="about-this-guide.html#other-places-to-find-information">"About this guide"</a> chapter.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="about-the-compiler-team"><a class="header" href="#about-the-compiler-team">About the compiler team</a></h1> |
| <blockquote> |
| <p>NOTE: |
| There exists much detail about the team <a href="https://forge.rust-lang.org/compiler">on Forge</a>, making most of the following obsolete.</p> |
| </blockquote> |
| <p>rustc is maintained by the <a href="https://www.rust-lang.org/governance/teams/compiler">Rust compiler team</a>. The people who belong to |
| this team collectively work to track regressions and implement new features. |
| Members of the Rust compiler team are people who have made significant |
| contributions to rustc and its design.</p> |
| <h2 id="discussion"><a class="header" href="#discussion">Discussion</a></h2> |
| <p>Currently the compiler team chats in Zulip:</p> |
| <ul> |
| <li>Team chat occurs in the <a href="https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler"><code>t-compiler</code></a> stream on the Zulip instance</li> |
| <li>There are also a number of other associated Zulip channels, |
| such as <a href="https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp"><code>t-compiler/help</code></a>, where people can ask for help |
| with rustc development, or <a href="https://rust-lang.zulipchat.com/#narrow/stream/238009-t-compiler.2Fmeetings"><code>t-compiler/meetings</code></a>, |
| where the team holds their weekly triage and steering meetings.</li> |
| </ul> |
| <h2 id="reviewers"><a class="header" href="#reviewers">Reviewers</a></h2> |
| <p>If you're interested in figuring out who can answer questions about a |
| particular part of the compiler, or you'd just like to know who works on what, |
| check out <a href="https://github.com/rust-lang/rust/blob/HEAD/triagebot.toml">triagebot.toml's assign section</a>. |
| It contains a listing of the various parts of the compiler and a list of people |
| who are reviewers of each part.</p> |
| <h2 id="rust-compiler-meeting"><a class="header" href="#rust-compiler-meeting">Rust compiler meeting</a></h2> |
| <p>The compiler team has a weekly meeting where we do triage and try to |
| generally stay on top of new bugs, regressions, and discuss important |
| things in general. |
| They are held on <a href="https://rust-lang.zulipchat.com/#narrow/stream/238009-t-compiler.2Fmeetings">Zulip</a>. It works roughly as follows:</p> |
| <ul> |
| <li><strong>Announcements, MCPs/FCPs, and WG-check-ins:</strong> We share some |
| announcements with the rest of the team about important things we want |
| everyone to be aware of. We also share the status of MCPs and FCPs and we |
| use the opportunity to have a couple of WGs giving us an update about |
| their work.</li> |
| <li><strong>Check for beta and stable nominations:</strong> These are nominations of things to |
| backport to beta and stable respectively. |
| We then look for new cases where the compiler broke previously working |
| code in the wild. Regressions are important issues to fix, so it's |
| likely that they are tagged as P-critical or P-high; the major |
| exception would be bug fixes (though even there we often <a href="./bug-fix-procedure.html">aim to give |
| warnings first</a>).</li> |
| <li><strong>Review P-critical and P-high bugs:</strong> P-critical and P-high bugs are |
| those that are sufficiently important for us to actively track |
| progress. P-critical and P-high bugs should ideally always have an |
| assignee.</li> |
| <li><strong>Check <code>S-waiting-on-t-compiler</code> and <code>I-compiler-nominated</code> issues:</strong> These are issues where |
| feedback from the team is desired.</li> |
| <li><strong>Look over the performance triage report:</strong> We check for PRs that made the |
| performance worse and try to decide if it's worth reverting the performance regression or if |
| the regression can be addressed in a future PR.</li> |
| </ul> |
| <p>The meeting currently takes place on Thursdays at 10am Boston time |
| (UTC-4 typically, but daylight savings time sometimes makes things |
| complicated).</p> |
| <h2 id="team-membership"><a class="header" href="#team-membership">Team membership</a></h2> |
| <p>Membership in the Rust team is typically offered when someone has been |
| making significant contributions to the compiler for some |
| time. Membership is both a recognition but also an obligation: |
| compiler team members are generally expected to help with upkeep as |
| well as doing reviews and other work.</p> |
| <p>If you are interested in becoming a compiler team member, the first |
| thing to do is to start fixing some bugs, or get involved in a working |
| group. One good way to find bugs is to look for |
| <a href="https://github.com/rust-lang/rust/issues?q=is%3Aopen+is%3Aissue+label%3AE-easy">open issues tagged with E-easy</a> |
| or |
| <a href="https://github.com/rust-lang/rust/issues?q=is%3Aopen+is%3Aissue+label%3AE-mentor">E-mentor</a>.</p> |
| <p>You can also dig through the graveyard of PRs that were |
| <a href="https://github.com/rust-lang/rust/pulls?q=is%3Apr+label%3AS-inactive">closed due to inactivity</a>, |
| some of them may contain work that is still useful - refer to the |
| associated issues, if any - and only needs some finishing touches |
| for which the original author didn't have time.</p> |
| <h3 id="r-rights"><a class="header" href="#r-rights">r+ rights</a></h3> |
| <p>Once you have made a number of individual PRs to rustc, we will often |
| offer r+ privileges. This means that you have the right to instruct |
| "bors" (the robot that manages which PRs get landed into rustc) to |
| merge a PR |
| (<a href="https://bors.rust-lang.org/">here are some instructions for how to talk to bors</a>).</p> |
| <p>The guidelines for reviewers are as follows:</p> |
| <ul> |
| <li>You are always welcome to review any PR, regardless of who it is |
| assigned to. However, do not r+ PRs unless: |
| <ul> |
| <li>You are confident in that part of the code.</li> |
| <li>You are confident that nobody else wants to review it first. |
| <ul> |
| <li>For example, sometimes people will express a desire to review a |
| PR before it lands, perhaps because it touches a particularly |
| sensitive part of the code.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>Always be polite when reviewing: you are a representative of the |
| Rust project, so it is expected that you will go above and beyond |
| when it comes to the <a href="https://www.rust-lang.org/policies/code-of-conduct">Code of Conduct</a>.</li> |
| </ul> |
| <h3 id="reviewer-rotation"><a class="header" href="#reviewer-rotation">Reviewer rotation</a></h3> |
| <p>Once you have r+ rights, you can also be added to the <a href="https://github.com/rust-lang/rust/blob/36285c5de8915ecc00d91ae0baa79a87ed5858d5/triagebot.toml#L528-L577">reviewer rotation</a>. |
| <a href="https://github.com/rust-lang/triagebot/">triagebot</a> is the bot that <a href="https://forge.rust-lang.org/triagebot/pr-assignment.html">automatically assigns</a> incoming PRs to reviewers. |
| If you are added, you will be randomly selected to review |
| PRs. If you find you are assigned a PR that you don't feel comfortable |
| reviewing, you can also leave a comment like <code>r? @so-and-so</code> to assign |
| to someone else — if you don't know who to request, just write <code>r? @nikomatsakis for reassignment</code> and @nikomatsakis will pick someone |
| for you.</p> |
| <p>Getting on the reviewer rotation is much appreciated as it lowers the |
| review burden for all of us! However, if you don't have time to give |
| people timely feedback on their PRs, it may be better that you don't |
| get on the list.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="using-git"><a class="header" href="#using-git">Using Git</a></h1> |
| <p>The Rust project uses <a href="https://git-scm.com">Git</a> to manage its source code. In order to |
| contribute, you'll need some familiarity with its features so that your changes |
| can be incorporated into the compiler.</p> |
| <p>The goal of this page is to cover some of the more common questions and |
| problems new contributors face. Although some Git basics will be covered here, |
| if you find that this is still a little too fast for you, it might make sense |
| to first read some introductions to Git, such as the Beginner and Getting |
| started sections of <a href="https://www.atlassian.com/git/tutorials/what-is-version-control">this tutorial from Atlassian</a>. GitHub also |
| provides <a href="https://docs.github.com/en/get-started/quickstart/set-up-git">documentation</a> and <a href="https://guides.github.com/introduction/git-handbook/">guides</a> for beginners, or you can consult the |
| more in depth <a href="https://git-scm.com/book/en/v2/">book from Git</a>.</p> |
| <p>This guide is incomplete. If you run into trouble with git that this page doesn't help with, |
| please <a href="https://github.com/rust-lang/rustc-dev-guide/issues/new">open an issue</a> so we can document how to fix it.</p> |
| <h2 id="prerequisites-1"><a class="header" href="#prerequisites-1">Prerequisites</a></h2> |
| <p>We'll assume that you've installed Git, forked <a href="https://github.com/rust-lang/rust">rust-lang/rust</a>, and cloned the |
| forked repo to your PC. We'll use the command line interface to interact |
| with Git; there are also a number of GUIs and IDE integrations that can |
| generally do the same things.</p> |
| <p>If you've cloned your fork, then you will be able to reference it with <code>origin</code> |
| in your local repo. It may be helpful to also set up a remote for the official |
| rust-lang/rust repo via</p> |
| <pre><code class="language-console">git remote add upstream https://github.com/rust-lang/rust.git |
| </code></pre> |
| <p>if you're using HTTPS, or</p> |
| <pre><code class="language-console">git remote add upstream git@github.com:rust-lang/rust.git |
| </code></pre> |
| <p>if you're using SSH.</p> |
| <p><strong>NOTE:</strong> This page is dedicated to workflows for <code>rust-lang/rust</code>, but will likely be |
| useful when contributing to other repositories in the Rust project.</p> |
| <h2 id="standard-process"><a class="header" href="#standard-process">Standard Process</a></h2> |
| <p>Below is the normal procedure that you're likely to use for most minor changes |
| and PRs:</p> |
| <ol> |
| <li>Ensure that you're making your changes on top of <code>main</code>: |
| <code>git checkout main</code>.</li> |
| <li>Get the latest changes from the Rust repo: <code>git pull upstream main --ff-only</code>. |
| (see <a href="git.html#keeping-things-up-to-date">No-Merge Policy</a> for more info about this).</li> |
| <li>Make a new branch for your change: <code>git checkout -b issue-12345-fix</code>.</li> |
| <li>Make some changes to the repo and test them.</li> |
| <li>Stage your changes via <code>git add src/changed/file.rs src/another/change.rs</code> |
| and then commit them with <code>git commit</code>. Of course, making intermediate commits |
| may be a good idea as well. Avoid <code>git add .</code>, as it makes it too easy to |
| unintentionally commit changes that should not be committed, such as submodule |
| updates. You can use <code>git status</code> to check if there are any files you forgot |
| to stage.</li> |
| <li>Push your changes to your fork: <code>git push --set-upstream origin issue-12345-fix</code> |
| (After adding commits, you can use <code>git push</code> and after rebasing or |
| pulling-and-rebasing, you can use <code>git push --force-with-lease</code>).</li> |
| <li><a href="https://guides.github.com/activities/forking/#making-a-pull-request">Open a PR</a> from your fork to <code>rust-lang/rust</code>'s <code>main</code> branch.</li> |
| </ol> |
| <p>If you end up needing to rebase and are hitting conflicts, see <a href="git.html#rebasing">Rebasing</a>. |
| If you want to track upstream while working on long-running feature/issue, see |
| <a href="git.html#keeping-things-up-to-date">Keeping things up to date</a>.</p> |
| <p>If your reviewer requests changes, the procedure for those changes looks much |
| the same, with some steps skipped:</p> |
| <ol> |
| <li>Ensure that you're making changes to the most recent version of your code: |
| <code>git checkout issue-12345-fix</code>.</li> |
| <li>Make, stage, and commit your additional changes just like before.</li> |
| <li>Push those changes to your fork: <code>git push</code>.</li> |
| </ol> |
| <h2 id="troubleshooting-git-issues"><a class="header" href="#troubleshooting-git-issues">Troubleshooting git issues</a></h2> |
| <p>You don't need to clone <code>rust-lang/rust</code> from scratch if it's out of date! |
| Even if you think you've messed it up beyond repair, there are ways to fix |
| the git state that don't require downloading the whole repository again. |
| Here are some common issues you might run into:</p> |
| <h3 id="i-made-a-merge-commit-by-accident"><a class="header" href="#i-made-a-merge-commit-by-accident">I made a merge commit by accident.</a></h3> |
| <p>Git has two ways to update your branch with the newest changes: merging and rebasing. |
| Rust <a href="git.html#keeping-things-up-to-date">uses rebasing</a>. If you make a merge commit, it's not too hard to fix: |
| <code>git rebase -i upstream/main</code>.</p> |
| <p>See <a href="git.html#rebasing">Rebasing</a> for more about rebasing.</p> |
| <h3 id="i-deleted-my-fork-on-github"><a class="header" href="#i-deleted-my-fork-on-github">I deleted my fork on GitHub!</a></h3> |
| <p>This is not a problem from git's perspective. If you run <code>git remote -v</code>, |
| it will say something like this:</p> |
| <pre><code class="language-console">$ git remote -v |
| origin git@github.com:jyn514/rust.git (fetch) |
| origin git@github.com:jyn514/rust.git (push) |
| upstream https://github.com/rust-lang/rust (fetch) |
| upstream https://github.com/rust-lang/rust (fetch) |
| </code></pre> |
| <p>If you renamed your fork, you can change the URL like this:</p> |
| <pre><code class="language-console">git remote set-url origin <URL> |
| </code></pre> |
| <p>where the <code><URL></code> is your new fork.</p> |
| <h3 id="i-changed-a-submodule-by-accident"><a class="header" href="#i-changed-a-submodule-by-accident">I changed a submodule by accident</a></h3> |
| <p>Usually people notice this when rustbot posts a comment on github that <code>cargo</code> has been modified:</p> |
| <p><img src="./img/rustbot-submodules.png" alt="rustbot submodule comment" /></p> |
| <p>You might also notice conflicts in the web UI:</p> |
| <p><img src="./img/submodule-conflicts.png" alt="conflict in src/tools/cargo" /></p> |
| <p>The most common cause is that you rebased after a change and ran <code>git add .</code> without first running |
| <code>x</code> to update the submodules. Alternatively, you might have run <code>cargo fmt</code> instead of <code>x fmt</code> |
| and modified files in a submodule, then committed the changes.</p> |
| <p>To fix it, do the following things (if you changed a submodule other than cargo, |
| replace <code>src/tools/cargo</code> with the path to that submodule):</p> |
| <ol> |
| <li>See which commit has the accidental changes: <code>git log --stat -n1 src/tools/cargo</code></li> |
| <li>Revert the changes to that commit: <code>git checkout <my-commit>~ src/tools/cargo</code>. Type <code>~</code> |
| literally but replace <code><my-commit></code> with the output from step 1.</li> |
| <li>Tell git to commit the changes: <code>git commit --fixup <my-commit></code></li> |
| <li>Repeat steps 1-3 for all the submodules you modified. |
| <ul> |
| <li>If you modified the submodule in several different commits, you will need to repeat steps 1-3 |
| for each commit you modified. You'll know when to stop when the <code>git log</code> command shows a commit |
| that's not authored by you.</li> |
| </ul> |
| </li> |
| <li>Squash your changes into the existing commits: <code>git rebase --autosquash -i upstream/main</code></li> |
| <li><a href="git.html#standard-process">Push your changes</a>.</li> |
| </ol> |
| <h3 id="i-see-error-cannot-rebase-when-i-try-to-rebase"><a class="header" href="#i-see-error-cannot-rebase-when-i-try-to-rebase">I see "error: cannot rebase" when I try to rebase</a></h3> |
| <p>These are two common errors to see when rebasing:</p> |
| <pre><code class="language-console">error: cannot rebase: Your index contains uncommitted changes. |
| error: Please commit or stash them. |
| </code></pre> |
| <pre><code class="language-console">error: cannot rebase: You have unstaged changes. |
| error: Please commit or stash them. |
| </code></pre> |
| <p>(See <a href="https://git-scm.com/book/en/v2/Getting-Started-What-is-Git%3F#_the_three_states">https://git-scm.com/book/en/v2/Getting-Started-What-is-Git%3F#_the_three_states</a> for the difference between the two.)</p> |
| <p>This means you have made changes since the last time you made a commit. To be able to rebase, either |
| commit your changes, or make a temporary commit called a "stash" to have them still not be committed |
| when you finish rebasing. You may want to configure git to make this "stash" automatically, which |
| will prevent the "cannot rebase" error in nearly all cases:</p> |
| <pre><code class="language-console">git config --global rebase.autostash true |
| </code></pre> |
| <p>See <a href="https://git-scm.com/book/en/v2/Git-Tools-Stashing-and-Cleaning">https://git-scm.com/book/en/v2/Git-Tools-Stashing-and-Cleaning</a> for more info about stashing.</p> |
| <h3 id="i-see-untracked-files-srcstdarch"><a class="header" href="#i-see-untracked-files-srcstdarch">I see 'Untracked Files: src/stdarch'?</a></h3> |
| <p>This is left over from the move to the <code>library/</code> directory. |
| Unfortunately, <code>git rebase</code> does not follow renames for submodules, so you |
| have to delete the directory yourself:</p> |
| <pre><code class="language-console">rm -r src/stdarch |
| </code></pre> |
| <h3 id="i-see--head"><a class="header" href="#i-see--head">I see <code><<< HEAD</code>?</a></h3> |
| <p>You were probably in the middle of a rebase or merge conflict. See |
| <a href="git.html#rebasing-and-conflicts">Conflicts</a> for how to fix the conflict. If you don't care about the changes |
| and just want to get a clean copy of the repository back, you can use <code>git reset</code>:</p> |
| <pre><code class="language-console"># WARNING: this throws out any local changes you've made! Consider resolving the conflicts instead. |
| git reset --hard main |
| </code></pre> |
| <h3 id="failed-to-push-some-refs"><a class="header" href="#failed-to-push-some-refs">failed to push some refs</a></h3> |
| <p><code>git push</code> will not work properly and say something like this:</p> |
| <pre><code class="language-console"> ! [rejected] issue-xxxxx -> issue-xxxxx (non-fast-forward) |
| error: failed to push some refs to 'https://github.com/username/rust.git' |
| hint: Updates were rejected because the tip of your current branch is behind |
| hint: its remote counterpart. Integrate the remote changes (e.g. |
| hint: 'git pull ...') before pushing again. |
| hint: See the 'Note about fast-forwards' in 'git push --help' for details. |
| </code></pre> |
| <p>The advice this gives is incorrect! Because of Rust's |
| <a href="git.html#no-merge-policy">"no-merge" policy</a> the merge commit created by <code>git pull</code> |
| will not be allowed in the final PR, in addition to defeating the point of the |
| rebase! Use <code>git push --force-with-lease</code> instead.</p> |
| <h3 id="git-is-trying-to-rebase-commits-i-didnt-write"><a class="header" href="#git-is-trying-to-rebase-commits-i-didnt-write">Git is trying to rebase commits I didn't write?</a></h3> |
| <p>If you see many commits in your rebase list, or merge commits, or commits by other people that you |
| didn't write, it likely means you're trying to rebase over the wrong branch. For example, you may |
| have a <code>rust-lang/rust</code> remote <code>upstream</code>, but ran <code>git rebase origin/main</code> instead of <code>git rebase upstream/main</code>. The fix is to abort the rebase and use the correct branch instead:</p> |
| <pre><code class="language-console">git rebase --abort |
| git rebase --interactive upstream/main |
| </code></pre> |
| <details><summary>Click here to see an example of rebasing over the wrong branch</summary> |
| <p><img src="img/other-peoples-commits.png" alt="Interactive rebase over the wrong branch" /></p> |
| </details> |
| <h3 id="quick-note-about-submodules"><a class="header" href="#quick-note-about-submodules">Quick note about submodules</a></h3> |
| <p>When updating your local repository with <code>git pull</code>, you may notice that sometimes |
| Git says you have modified some files that you have never edited. For example, |
| running <code>git status</code> gives you something like (note the <code>new commits</code> mention):</p> |
| <pre><code class="language-console">On branch main |
| Your branch is up to date with 'origin/main'. |
| |
| Changes not staged for commit: |
| (use "git add <file>..." to update what will be committed) |
| (use "git restore <file>..." to discard changes in working directory) |
| modified: src/llvm-project (new commits) |
| modified: src/tools/cargo (new commits) |
| |
| no changes added to commit (use "git add" and/or "git commit -a") |
| </code></pre> |
| <p>These changes are not changes to files: they are changes to submodules (more on this <a href="git.html#git-submodules">later</a>). |
| To get rid of those:</p> |
| <pre><code class="language-console">git submodule update |
| </code></pre> |
| <p>Some submodules are not actually needed; for example, <code>src/llvm-project</code> doesn't need to be checked |
| out if you're using <code>download-ci-llvm</code>. To avoid having to keep fetching its history, you can use |
| <code>git submodule deinit -f src/llvm-project</code>, which will also avoid it showing as modified again.</p> |
| <h2 id="rebasing-and-conflicts"><a class="header" href="#rebasing-and-conflicts">Rebasing and Conflicts</a></h2> |
| <p>When you edit your code locally, you are making changes to the version of |
| rust-lang/rust that existed when you created your feature branch. As such, when |
| you submit your PR it is possible that some of the changes that have been made |
| to rust-lang/rust since then are in conflict with the changes you've made. |
| When this happens, you need to resolve the conflicts before your changes can be |
| merged. To do that, you need to rebase your work on top of rust-lang/rust.</p> |
| <h3 id="rebasing"><a class="header" href="#rebasing">Rebasing</a></h3> |
| <p>To rebase your feature branch on top of the newest version of the <code>main</code> branch |
| of rust-lang/rust, checkout your branch, and then run this command:</p> |
| <pre><code class="language-console">git pull --rebase https://github.com/rust-lang/rust.git main |
| </code></pre> |
| <blockquote> |
| <p>If you are met with the following error:</p> |
| <pre><code class="language-console">error: cannot pull with rebase: Your index contains uncommitted changes. |
| error: please commit or stash them. |
| </code></pre> |
| <p>it means that you have some uncommitted work in your working tree. In that |
| case, run <code>git stash</code> before rebasing, and then <code>git stash pop</code> after you |
| have rebased and fixed all conflicts.</p> |
| </blockquote> |
| <p>When you rebase a branch on main, all the changes on your branch are |
| reapplied to the most recent version of <code>main</code>. In other words, Git tries to |
| pretend that the changes you made to the old version of <code>main</code> were instead |
| made to the new version of <code>main</code>. During this process, you should expect to |
| encounter at least one "rebase conflict." This happens when Git's attempt to |
| reapply the changes fails because your changes conflicted with other changes |
| that have been made. You can tell that this happened because you'll see |
| lines in the output that look like</p> |
| <pre><code class="language-console">CONFLICT (content): Merge conflict in file.rs |
| </code></pre> |
| <p>When you open these files, you'll see sections of the form</p> |
| <pre><code class="language-console"><<<<<<< HEAD |
| Original code |
| ======= |
| Your code |
| >>>>>>> 8fbf656... Commit fixes 12345 |
| </code></pre> |
| <p>This represents the lines in the file that Git could not figure out how to |
| rebase. The section between <code><<<<<<< HEAD</code> and <code>=======</code> has the code from |
| <code>main</code>, while the other side has your version of the code. You'll need to |
| decide how to deal with the conflict. You may want to keep your changes, |
| keep the changes on <code>main</code>, or combine the two.</p> |
| <p>Generally, resolving the conflict consists of two steps: First, fix the |
| particular conflict. Edit the file to make the changes you want and remove the |
| <code><<<<<<<</code>, <code>=======</code> and <code>>>>>>>></code> lines in the process. Second, check the |
| surrounding code. If there was a conflict, its likely there are some logical |
| errors lying around too! It's a good idea to run <code>x check</code> here to make sure |
| there are no glaring errors.</p> |
| <p>Once you're all done fixing the conflicts, you need to stage the files that had |
| conflicts in them via <code>git add</code>. Afterwards, run <code>git rebase --continue</code> to let |
| Git know that you've resolved the conflicts and it should finish the rebase.</p> |
| <p>Once the rebase has succeeded, you'll want to update the associated branch on |
| your fork with <code>git push --force-with-lease</code>.</p> |
| <h3 id="keeping-things-up-to-date"><a class="header" href="#keeping-things-up-to-date">Keeping things up to date</a></h3> |
| <p>The <a href="git.html#rebasing">above section</a> is a specific |
| guide on rebasing work and dealing with merge conflicts. |
| Here is some general advice about how to keep your local repo |
| up-to-date with upstream changes:</p> |
| <p>Using <code>git pull upstream main</code> while on your local <code>main</code> branch regularly |
| will keep it up-to-date. You will also want to keep your feature branches |
| up-to-date as well. After pulling, you can checkout the feature branches |
| and rebase them:</p> |
| <pre><code class="language-console">git checkout main |
| git pull upstream main --ff-only # to make certain there are no merge commits |
| git rebase main feature_branch |
| git push --force-with-lease # (set origin to be the same as local) |
| </code></pre> |
| <p>To avoid merges as per the <a href="git.html#no-merge-policy">No-Merge Policy</a>, you may want to use |
| <code>git config pull.ff only</code> (this will apply the config only to the local repo) |
| to ensure that Git doesn't create merge commits when <code>git pull</code>ing, without |
| needing to pass <code>--ff-only</code> or <code>--rebase</code> every time.</p> |
| <p>You can also <code>git push --force-with-lease</code> from main to double-check that your |
| feature branches are in sync with their state on the Github side.</p> |
| <h2 id="advanced-rebasing"><a class="header" href="#advanced-rebasing">Advanced Rebasing</a></h2> |
| <h3 id="squash-your-commits"><a class="header" href="#squash-your-commits">Squash your commits</a></h3> |
| <p>"Squashing" commits into each other causes them to be merged into a single |
| commit. Both the upside and downside of this is that it simplifies the history. |
| On the one hand, you lose track of the steps in which changes were made, but |
| the history becomes easier to work with.</p> |
| <p>If there are no conflicts and you are just squashing to clean up the history, |
| use <code>git rebase --interactive --keep-base main</code>. This keeps the fork point |
| of your PR the same, making it easier to review the diff of what happened |
| across your rebases.</p> |
| <p>Squashing can also be useful as part of conflict resolution. |
| If your branch contains multiple consecutive rewrites of the same code, or if |
| the rebase conflicts are extremely severe, you can use |
| <code>git rebase --interactive main</code> to gain more control over the process. This |
| allows you to choose to skip commits, edit the commits that you do not skip, |
| change the order in which they are applied, or "squash" them into each other.</p> |
| <p>Alternatively, you can sacrifice the commit history like this:</p> |
| <pre><code class="language-console"># squash all the changes into one commit so you only have to worry about conflicts once |
| git rebase --interactive --keep-base main # and squash all changes along the way |
| git rebase main |
| # fix all merge conflicts |
| git rebase --continue |
| </code></pre> |
| <p>You also may want to squash just the last few commits together, possibly |
| because they only represent "fixups" and not real changes. For example, |
| <code>git rebase --interactive HEAD~2</code> will allow you to edit the two commits only.</p> |
| <h3 id="git-range-diff"><a class="header" href="#git-range-diff"><code>git range-diff</code></a></h3> |
| <p>After completing a rebase, and before pushing up your changes, you may want to |
| review the changes between your old branch and your new one. You can do that |
| with <code>git range-diff main @{upstream} HEAD</code>.</p> |
| <p>The first argument to <code>range-diff</code>, <code>main</code> in this case, is the base revision |
| that you're comparing your old and new branch against. The second argument is |
| the old version of your branch; in this case, <code>@upstream</code> means the version that |
| you've pushed to GitHub, which is the same as what people will see in your pull |
| request. Finally, the third argument to <code>range-diff</code> is the <em>new</em> version of |
| your branch; in this case, it is <code>HEAD</code>, which is the commit that is currently |
| checked-out in your local repo.</p> |
| <p>Note that you can also use the equivalent, abbreviated form <code>git range-diff main @{u} HEAD</code>.</p> |
| <p>Unlike in regular Git diffs, you'll see a <code>-</code> or <code>+</code> next to another <code>-</code> or <code>+</code> |
| in the range-diff output. The marker on the left indicates a change between the |
| old branch and the new branch, and the marker on the right indicates a change |
| you've committed. So, you can think of a range-diff as a "diff of diffs" since |
| it shows you the differences between your old diff and your new diff.</p> |
| <p>Here's an example of <code>git range-diff</code> output (taken from <a href="https://git-scm.com/docs/git-range-diff#_examples">Git's |
| docs</a>):</p> |
| <pre><code class="language-console">-: ------- > 1: 0ddba11 Prepare for the inevitable! |
| 1: c0debee = 2: cab005e Add a helpful message at the start |
| 2: f00dbal ! 3: decafe1 Describe a bug |
| @@ -1,3 +1,3 @@ |
| Author: A U Thor <author@example.com> |
| |
| -TODO: Describe a bug |
| +Describe a bug |
| @@ -324,5 +324,6 |
| This is expected. |
| |
| -+What is unexpected is that it will also crash. |
| ++Unexpectedly, it also crashes. This is a bug, and the jury is |
| ++still out there how to fix it best. See ticket #314 for details. |
| |
| Contact |
| 3: bedead < -: ------- TO-UNDO |
| </code></pre> |
| <p>(Note that <code>git range-diff</code> output in your terminal will probably be easier to |
| read than in this example because it will have colors.)</p> |
| <p>Another feature of <code>git range-diff</code> is that, unlike <code>git diff</code>, it will also |
| diff commit messages. This feature can be useful when amending several commit |
| messages so you can make sure you changed the right parts.</p> |
| <p><code>git range-diff</code> is a very useful command, but note that it can take some time |
| to get used to its output format. You may also find Git's documentation on the |
| command useful, especially their <a href="https://git-scm.com/docs/git-range-diff#_examples">"Examples" section</a>.</p> |
| <h2 id="no-merge-policy"><a class="header" href="#no-merge-policy">No-Merge Policy</a></h2> |
| <p>The rust-lang/rust repo uses what is known as a "rebase workflow." This means |
| that merge commits in PRs are not accepted. As a result, if you are running |
| <code>git merge</code> locally, chances are good that you should be rebasing instead. Of |
| course, this is not always true; if your merge will just be a fast-forward, |
| like the merges that <code>git pull</code> usually performs, then no merge commit is |
| created and you have nothing to worry about. Running <code>git config merge.ff only</code> |
| (this will apply the config to the local repo) |
| once will ensure that all the merges you perform are of this type, so that you |
| cannot make a mistake.</p> |
| <p>There are a number of reasons for this decision and like all others, it is a |
| tradeoff. The main advantage is the generally linear commit history. This |
| greatly simplifies bisecting and makes the history and commit log much easier |
| to follow and understand.</p> |
| <h2 id="tips-for-reviewing"><a class="header" href="#tips-for-reviewing">Tips for reviewing</a></h2> |
| <p><strong>NOTE</strong>: This section is for <em>reviewing</em> PRs, not authoring them.</p> |
| <h3 id="hiding-whitespace"><a class="header" href="#hiding-whitespace">Hiding whitespace</a></h3> |
| <p>Github has a button for disabling whitespace changes that may be useful. |
| You can also use <code>git diff -w origin/main</code> to view changes locally.</p> |
| <p><img src="./img/github-whitespace-changes.png" alt="hide whitespace" /></p> |
| <h3 id="fetching-prs"><a class="header" href="#fetching-prs">Fetching PRs</a></h3> |
| <p>To checkout PRs locally, you can use <code>git fetch upstream pull/NNNNN/head && git checkout FETCH_HEAD</code>.</p> |
| <p>You can also use github's cli tool. Github shows a button on PRs where you can copy-paste the |
| command to check it out locally. See <a href="https://cli.github.com/">https://cli.github.com/</a> for more info.</p> |
| <p><img src="./img/github-cli.png" alt="gh suggestion" /></p> |
| <h3 id="moving-large-sections-of-code"><a class="header" href="#moving-large-sections-of-code">Moving large sections of code</a></h3> |
| <p>Git and Github's default diff view for large moves <em>within</em> a file is quite poor; it will show each |
| line as deleted and each line as added, forcing you to compare each line yourself. Git has an option |
| to show moved lines in a different color:</p> |
| <pre><code class="language-console">git log -p --color-moved=dimmed-zebra --color-moved-ws=allow-indentation-change |
| </code></pre> |
| <p>See <a href="https://git-scm.com/docs/git-diff#Documentation/git-diff.txt---color-movedltmodegt">the docs for <code>--color-moved</code></a> for more info.</p> |
| <h3 id="range-diff"><a class="header" href="#range-diff">range-diff</a></h3> |
| <p>See <a href="git.html#git-range-diff">the relevant section for PR authors</a>. This can be useful for comparing code |
| that was force-pushed to make sure there are no unexpected changes.</p> |
| <h3 id="ignoring-changes-to-specific-files"><a class="header" href="#ignoring-changes-to-specific-files">Ignoring changes to specific files</a></h3> |
| <p>Many large files in the repo are autogenerated. To view a diff that ignores changes to those files, |
| you can use the following syntax (e.g. Cargo.lock):</p> |
| <pre><code class="language-console">git log -p ':!Cargo.lock' |
| </code></pre> |
| <p>Arbitrary patterns are supported (e.g. <code>:!compiler/*</code>). Patterns use the same syntax as |
| <code>.gitignore</code>, with <code>:</code> prepended to indicate a pattern.</p> |
| <h2 id="git-submodules"><a class="header" href="#git-submodules">Git submodules</a></h2> |
| <p><strong>NOTE</strong>: submodules are a nice thing to know about, but it <em>isn't</em> an absolute |
| prerequisite to contribute to <code>rustc</code>. If you are using Git for the first time, |
| you might want to get used to the main concepts of Git before reading this section.</p> |
| <p>The <code>rust-lang/rust</code> repository uses <a href="https://git-scm.com/book/en/v2/Git-Tools-Submodules">Git submodules</a> as a way to use other |
| Rust projects from within the <code>rust</code> repo. Examples include Rust's fork of |
| <code>llvm-project</code>, <code>cargo</code> and libraries like <code>stdarch</code> and <code>backtrace</code>.</p> |
| <p>Those projects are developed and maintained in an separate Git (and GitHub) |
| repository, and they have their own Git history/commits, issue tracker and PRs. |
| Submodules allow us to create some sort of embedded sub-repository inside the |
| <code>rust</code> repository and use them like they were directories in the <code>rust</code> repository.</p> |
| <p>Take <code>llvm-project</code> for example. <code>llvm-project</code> is maintained in the <a href="https://github.com/rust-lang/llvm-project"><code>rust-lang/llvm-project</code></a> |
| repository, but it is used in <code>rust-lang/rust</code> by the compiler for code generation and |
| optimization. We bring it in <code>rust</code> as a submodule, in the <code>src/llvm-project</code> folder.</p> |
| <p>The contents of submodules are ignored by Git: submodules are in some sense isolated |
| from the rest of the repository. However, if you try to <code>cd src/llvm-project</code> and then |
| run <code>git status</code>:</p> |
| <pre><code class="language-console">HEAD detached at 9567f08afc943 |
| nothing to commit, working tree clean |
| </code></pre> |
| <p>As far as git is concerned, you are no longer in the <code>rust</code> repo, but in the <code>llvm-project</code> repo. |
| You will notice that we are in "detached HEAD" state, i.e. not on a branch but on a |
| particular commit.</p> |
| <p>This is because, like any dependency, we want to be able to control which version to use. |
| Submodules allow us to do just that: every submodule is "pinned" to a certain |
| commit, which doesn't change unless modified manually. If you use <code>git checkout <commit></code> |
| in the <code>llvm-project</code> directory and go back to the <code>rust</code> directory, you can stage this |
| change like any other, e.g. by running <code>git add src/llvm-project</code>. (Note that if |
| you <em>don't</em> stage the change to commit, then you run the risk that running |
| <code>x</code> will just undo your change by switching back to the previous commit when |
| it automatically "updates" the submodules.)</p> |
| <p>This version selection is usually done by the maintainers of the project, and |
| looks like <a href="https://github.com/rust-lang/rust/pull/99464/files">this</a>.</p> |
| <p>Git submodules take some time to get used to, so don't worry if it isn't perfectly |
| clear yet. You will rarely have to use them directly and, again, you don't need |
| to know everything about submodules to contribute to Rust. Just know that they |
| exist and that they correspond to some sort of embedded subrepository dependency |
| that Git can nicely and fairly conveniently handle for us.</p> |
| <h3 id="hard-resetting-submodules"><a class="header" href="#hard-resetting-submodules">Hard-resetting submodules</a></h3> |
| <p>Sometimes you might run into (when you run <code>git status</code>)</p> |
| <pre><code class="language-console">Changes not staged for commit: |
| (use "git add <file>..." to update what will be committed) |
| (use "git restore <file>..." to discard changes in working directory) |
| (commit or discard the untracked or modified content in submodules) |
| modified: src/llvm-project (new commits, modified content) |
| </code></pre> |
| <p>and when you try to run <code>git submodule update</code> it breaks horribly with errors like</p> |
| <pre><code class="language-console">error: RPC failed; curl 92 HTTP/2 stream 7 was not closed cleanly: CANCEL (err 8) |
| error: 2782 bytes of body are still expected |
| fetch-pack: unexpected disconnect while reading sideband packet |
| fatal: early EOF |
| fatal: fetch-pack: invalid index-pack output |
| fatal: Fetched in submodule path 'src/llvm-project', but it did not contain 5a5152f653959d14d68613a3a8a033fb65eec021. Direct fetching of that commit failed. |
| </code></pre> |
| <p>If you see <code>(new commits, modified content)</code> you can run</p> |
| <pre><code class="language-console">git submodule foreach git reset --hard |
| </code></pre> |
| <p>and then try <code>git submodule update</code> again.</p> |
| <h3 id="deinit-git-submodules"><a class="header" href="#deinit-git-submodules">Deinit git submodules</a></h3> |
| <p>If that doesn't work, you can try to deinit all git submodules...</p> |
| <pre><code class="language-console">git submodule deinit -f --all |
| </code></pre> |
| <p>Unfortunately sometimes your local git submodules configuration can become |
| completely messed up for some reason.</p> |
| <h3 id="overcoming-fatal-not-a-git-repository-submodulegitmodulessubmodule"><a class="header" href="#overcoming-fatal-not-a-git-repository-submodulegitmodulessubmodule">Overcoming <code>fatal: not a git repository: <submodule>/../../.git/modules/<submodule></code></a></h3> |
| <p>Sometimes, for some forsaken reason, you might run into</p> |
| <pre><code class="language-console">fatal: not a git repository: src/gcc/../../.git/modules/src/gcc |
| </code></pre> |
| <p>In this situation, for the given submodule path, i.e. <code><submodule_path> = src/gcc</code> in this example, you need to:</p> |
| <ol> |
| <li><code>rm -rf <submodule_path>/.git</code></li> |
| <li><code>rm -rf .git/modules/<submodule_path>/config</code></li> |
| <li><code>rm -rf .gitconfig.lock</code> if somehow the <code>.gitconfig</code> lock is orphaned.</li> |
| </ol> |
| <p>Then do something like <code>./x fmt</code> to have bootstrap manage the submodule |
| checkouts for you.</p> |
| <h2 id="ignoring-commits-during-git-blame"><a class="header" href="#ignoring-commits-during-git-blame">Ignoring commits during <code>git blame</code></a></h2> |
| <p>Some commits contain large reformatting changes that don't otherwise change functionality. They can |
| be instructed to be ignored by <code>git blame</code> through |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/.git-blame-ignore-revs"><code>.git-blame-ignore-revs</code></a>:</p> |
| <ol> |
| <li>Configure <code>git blame</code> to use <code>.git-blame-ignore-revs</code> as the list of commits to ignore: <code>git config blame.ignorerevsfile .git-blame-ignore-revs</code></li> |
| <li>Add suitable commits that you wish to be ignored by <code>git blame</code>.</li> |
| </ol> |
| <p>Please include a comment for the commit that you add to <code>.git-blame-ignore-revs</code> so people can |
| easily figure out <em>why</em> a commit is ignored.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mastering-rustbot"><a class="header" href="#mastering-rustbot">Mastering @rustbot</a></h1> |
| <p><code>@rustbot</code> (also known as <code>triagebot</code>) is a utility robot that is mostly used to |
| allow any contributor to achieve certain tasks that would normally require GitHub |
| membership to the <code>rust-lang</code> organization. Its most interesting features for |
| contributors to <code>rustc</code> are issue claiming and relabeling.</p> |
| <h2 id="issue-claiming"><a class="header" href="#issue-claiming">Issue claiming</a></h2> |
| <p><code>@rustbot</code> exposes a command that allows anyone to assign an issue to themselves. |
| If you see an issue you want to work on, you can send the following message as a |
| comment on the issue at hand:</p> |
| <pre><code>@rustbot claim |
| </code></pre> |
| <p>This will tell <code>@rustbot</code> to assign the issue to you if it has no assignee yet. |
| Note that because of some GitHub restrictions, you may be assigned indirectly, |
| i.e. <code>@rustbot</code> will assign itself as a placeholder and edit the top comment to |
| reflect the fact that the issue is now assigned to you.</p> |
| <p>If you want to unassign from an issue, <code>@rustbot</code> has a different command:</p> |
| <pre><code>@rustbot release-assignment |
| </code></pre> |
| <h2 id="issue-relabeling"><a class="header" href="#issue-relabeling">Issue relabeling</a></h2> |
| <p>Changing labels for an issue or PR is also normally reserved for members of the |
| organization. However, <code>@rustbot</code> allows you to relabel an issue yourself, only |
| with a few restrictions. This is mostly useful in two cases:</p> |
| <p><strong>Helping with issue triage</strong>: Rust's issue tracker has more than 5,000 open |
| issues at the time of this writing, so labels are the most powerful tool that we |
| have to keep it as tidy as possible. You don't need to spend hours in the issue tracker |
| to triage issues, but if you open an issue, you should feel free to label it if |
| you are comfortable with doing it yourself.</p> |
| <p><strong>Updating the status of a PR</strong>: We use "status labels" to reflect the status of |
| PRs. For example, if your PR has merge conflicts, it will automatically be assigned |
| the <code>S-waiting-on-author</code>, and reviewers might not review it until you rebase your |
| PR. Once you do rebase your branch, you should change the labels yourself to remove |
| the <code>S-waiting-on-author</code> label and add back <code>S-waiting-on-review</code>. In this case, |
| the <code>@rustbot</code> command will look like this:</p> |
| <pre><code>@rustbot label -S-waiting-on-author +S-waiting-on-review |
| </code></pre> |
| <p>The syntax for this command is pretty loose, so there are other variants of this |
| command invocation. There are also some shortcuts to update labels, |
| for instance <code>@rustbot ready</code> will do the same thing with above command. |
| For more details, see <a href="https://forge.rust-lang.org/triagebot/labeling.html">the docs page about labeling</a> and <a href="https://forge.rust-lang.org/triagebot/shortcuts.html">shortcuts</a>.</p> |
| <h2 id="other-commands"><a class="header" href="#other-commands">Other commands</a></h2> |
| <p>If you are interested in seeing what <code>@rustbot</code> is capable of, check out its <a href="https://forge.rust-lang.org/triagebot/index.html">documentation</a>, |
| which is meant as a reference for the bot and should be kept up to date every time the |
| bot gets an upgrade.</p> |
| <p><code>@rustbot</code> is maintained by the Release team. If you have any feedback regarding |
| existing commands or suggestions for new commands, feel free to reach out |
| <a href="https://rust-lang.zulipchat.com/#narrow/stream/224082-t-release.2Ftriagebot">on Zulip</a> or file an issue in <a href="https://github.com/rust-lang/triagebot/">the triagebot repository</a></p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="walkthrough-a-typical-contribution"><a class="header" href="#walkthrough-a-typical-contribution">Walkthrough: a typical contribution</a></h1> |
| <p>There are <em>a lot</em> of ways to contribute to the Rust compiler, including fixing |
| bugs, improving performance, helping design features, providing feedback on existing features, etc. |
| This chapter does not claim to scratch the surface. |
| Instead, it walks through the design and implementation of a new feature. |
| Not all of the steps and processes described here are needed for every |
| contribution, and I will try to point those out as they arise.</p> |
| <p>In general, if you are interested in making a contribution and aren't sure |
| where to start, please feel free to ask!</p> |
| <h2 id="overview"><a class="header" href="#overview">Overview</a></h2> |
| <p>The feature I will discuss in this chapter is the <code>?</code> Kleene operator for macros. |
| Basically, we want to be able to write something like this:</p> |
| <pre><code class="language-rust ignore">macro_rules! foo { |
| ($arg:ident $(, $optional_arg:ident)?) => { |
| println!("{}", $arg); |
| |
| $( |
| println!("{}", $optional_arg); |
| )? |
| } |
| } |
| |
| fn main() { |
| let x = 0; |
| foo!(x); // ok! prints "0" |
| foo!(x, x); // ok! prints "0 0" |
| }</code></pre> |
| <p>So basically, the <code>$(pat)?</code> matcher in the macro means "this pattern can occur |
| 0 or 1 times", similar to other regex syntaxes.</p> |
| <p>There were a number of steps to go from an idea to stable Rust feature. |
| Here is a quick list. |
| We will go through each of these in order below. |
| As I mentioned before, not all of these are needed for every type of contribution.</p> |
| <ul> |
| <li><strong>Idea discussion/Pre-RFC</strong> A Pre-RFC is an early draft or design discussion |
| of a feature. |
| This stage is intended to flesh out the design space a bit and |
| get a grasp on the different merits and problems with an idea. |
| It's a great way to get early feedback on your idea before presenting it to the wider |
| audience. |
| You can find the original discussion <a href="https://internals.rust-lang.org/t/pre-rfc-at-most-one-repetition-macro-patterns/6557">here</a>.</li> |
| <li><strong>RFC</strong> This is when you formally present your idea to the community for |
| consideration. |
| You can find the RFC <a href="https://github.com/rust-lang/rfcs/pull/2298">here</a>.</li> |
| <li><strong>Implementation</strong> Implement your idea unstably in the compiler. You can |
| find the original implementation <a href="https://github.com/rust-lang/rust/pull/47752">here</a>.</li> |
| <li><strong>Possibly iterate/refine</strong> As the community gets experience with your |
| feature on the nightly compiler and in <code>std</code>, there may be additional |
| feedback about design choice that might be adjusted. |
| This particular feature went <a href="https://github.com/rust-lang/rust/pull/49719">through</a> a <a href="https://github.com/rust-lang/rust/pull/51336">number</a> of <a href="https://github.com/rust-lang/rust/pull/51587">iterations</a>.</li> |
| <li><strong>Stabilization</strong> When your feature has baked enough, a Rust team member may |
| <a href="https://github.com/rust-lang/rust/issues/48075#issuecomment-433177613">propose to stabilize it</a>. |
| If there is consensus, this is done.</li> |
| <li><strong>Relax</strong> Your feature is now a stable Rust feature!</li> |
| </ul> |
| <h2 id="pre-rfc-and-rfc"><a class="header" href="#pre-rfc-and-rfc">Pre-RFC and RFC</a></h2> |
| <blockquote> |
| <p>NOTE: In general, if you are not proposing a <em>new</em> feature or substantial |
| change to Rust or the ecosystem, you don't need to follow the RFC process. |
| Instead, you can just jump to <a href="walkthrough.html#impl">implementation</a>.</p> |
| <p>You can find the official guidelines for when to open an RFC <a href="https://github.com/rust-lang/rfcs#when-you-need-to-follow-this-process">here</a>.</p> |
| </blockquote> |
| <p>An RFC is a document that describes the feature or change you are proposing in detail. |
| Anyone can write an RFC; |
| the process is the same for everyone, including Rust team members.</p> |
| <p>To open an RFC, open a PR on the <a href="https://github.com/rust-lang/rfcs">rust-lang/rfcs</a> repo on GitHub. |
| You can find detailed instructions in the |
| <a href="https://github.com/rust-lang/rfcs#what-the-process-is">README</a>.</p> |
| <p>Before opening an RFC, you should do the research to "flesh out" your idea. |
| Hastily-proposed RFCs tend not to be accepted. |
| You should generally have a good description of the motivation, impact, disadvantages, and potential |
| interactions with other features.</p> |
| <p>If that sounds like a lot of work, it's because it is. |
| But no fear! |
| Even if you're not a compiler hacker, you can get great feedback by doing a <em>pre-RFC</em>. |
| This is an <em>informal</em> discussion of the idea. |
| The best place to do this is internals.rust-lang.org. |
| Your post doesn't have to follow any particular structure. |
| It doesn't even need to be a cohesive idea. |
| Generally, you will get tons of feedback that you can integrate back to produce a good RFC.</p> |
| <p>(Another pro-tip: try searching the RFCs repo and internals for prior related ideas. |
| A lot of times an idea has already been considered and was either |
| rejected or postponed to be tried again later. |
| This can save you and everybody else some time)</p> |
| <p>In the case of our example, a participant in the pre-RFC thread pointed out a |
| syntax ambiguity and a potential resolution. |
| Also, the overall feedback seemed positive. |
| In this case, the discussion converged pretty quickly, but for some |
| ideas, a lot more discussion can happen (e.g. see <a href="https://github.com/rust-lang/rfcs/pull/2457">this RFC</a> which |
| received a whopping 684 comments!). |
| If that happens, don't be discouraged; |
| it means the community is interested in your idea, but it perhaps needs some |
| adjustments.</p> |
| <p>The RFC for our <code>?</code> macro feature did receive some discussion on the RFC thread too. |
| As with most RFCs, there were a few questions that we couldn't answer by |
| discussion: we needed experience using the feature to decide. |
| Such questions are listed in the "Unresolved Questions" section of the RFC. |
| Also, over the course of the RFC discussion, you will probably want to update the RFC document |
| itself to reflect the course of the discussion (e.g. new alternatives or prior |
| work may be added or you may decide to change parts of the proposal itself).</p> |
| <p>In the end, when the discussion seems to reach a consensus and die down a bit, |
| a Rust team member may propose to move to "final comment period" (FCP) with one |
| of three possible dispositions. |
| This means that they want the other members of |
| the appropriate teams to review and comment on the RFC. |
| More discussion may ensue, which may result in more changes or unresolved questions being added. |
| At some point, when everyone is satisfied, the RFC enters the FCP, which is the |
| last chance for people to bring up objections. |
| When the FCP is over, the disposition is adopted. |
| Here are the three possible dispositions:</p> |
| <ul> |
| <li><em>Merge</em>: accept the feature. Here is the proposal to merge for our <a href="https://github.com/rust-lang/rfcs/pull/2298#issuecomment-360582667"><code>?</code> macro |
| feature</a>.</li> |
| <li><em>Close</em>: this feature in its current form is not a good fit for rust. Don't |
| be discouraged if this happens to your RFC, and don't take it personally. |
| This is not a reflection on you, but rather a community decision that rust |
| will go a different direction.</li> |
| <li><em>Postpone</em>: there is interest in going this direction but not at the moment. |
| This happens most often because the appropriate Rust team doesn't have the |
| bandwidth to shepherd the feature through the process to stabilization. |
| Often this is the case when the feature doesn't fit into the team's roadmap. |
| Postponed ideas may be revisited later.</li> |
| </ul> |
| <p>When an RFC is merged, the PR is merged into the RFCs repo. |
| A new <em>tracking issue</em> is created in the <a href="https://github.com/rust-lang/rust">rust-lang/rust</a> repo to track progress on the feature |
| and discuss unresolved questions, implementation progress and blockers, etc. |
| Here is the tracking issue on for our <a href="https://github.com/rust-lang/rust/issues/48075"><code>?</code> macro feature</a>.</p> |
| <p><a id="impl"></a></p> |
| <h2 id="implementation"><a class="header" href="#implementation">Implementation</a></h2> |
| <p>To make a change to the compiler, open a PR against the <a href="https://github.com/rust-lang/rust">rust-lang/rust</a> repo.</p> |
| <p>Depending on the feature/change/bug fix/improvement, implementation may be |
| relatively-straightforward or it may be a major undertaking. |
| You can always ask for help or mentorship from more experienced compiler devs. |
| Also, you don't have to be the one to implement your feature; |
| but keep in mind that if you don't, it might be a while before someone else does.</p> |
| <p>For the <code>?</code> macro feature, I needed to go understand the relevant parts of |
| macro expansion in the compiler. |
| Personally, I find that <a href="https://github.com/rust-lang/rust/pull/47732">improving the |
| comments</a> in the code is a helpful way of making sure I understand |
| it, but you don't have to do that if you don't want to.</p> |
| <p>I then <a href="https://github.com/rust-lang/rust/pull/47752">implemented</a> the original feature, as described in the RFC. |
| When a new feature is implemented, it goes behind a <em>feature gate</em>, which means that |
| you have to use <code>#![feature(my_feature_name)]</code> to use the feature. |
| The feature gate is removed when the feature is stabilized.</p> |
| <p><strong>Most bug fixes and improvements</strong> don't require a feature gate. You can just |
| make your changes/improvements.</p> |
| <p>When you open a PR on the <a href="https://github.com/rust-lang/rust">rust-lang/rust</a>, a bot will assign your PR to a reviewer. |
| If there is a particular Rust team member you are working with, you can |
| request that reviewer by leaving a comment on the thread with <code>r? @reviewer-github-id</code> (e.g. <code>r? @eddyb</code>). If you don't know who to request, |
| don't request anyone; |
| the bot will assign someone automatically based on which files you changed.</p> |
| <p>The reviewer may request changes before they approve your PR, they may mark the PR with label |
| "S-waiting-on-author" after leaving comments, this means that the PR is blocked on you to make |
| some requested changes. |
| When you finished iterating on the changes, you can mark the PR as |
| <code>S-waiting-on-review</code> again by leaving a comment with <code>@rustbot ready</code>, this will remove the |
| <code>S-waiting-on-author</code> label and add the <code>S-waiting-on-review</code> label.</p> |
| <p>Feel free to ask questions or discuss things you don't understand or disagree with. |
| However, recognize that the PR won't be merged unless someone on the Rust team approves |
| it. |
| If a reviewer leave a comment like <code>r=me after fixing ...</code>, that means they approve the PR and |
| you can merge it with comment with <code>@bors r=reviewer-github-id</code>(e.g. <code>@bors r=eddyb</code>) to merge it |
| after fixing trivial issues. |
| Note that <code>r=someone</code> requires permission and bors could say |
| something like "🔑 Insufficient privileges..." when commenting <code>r=someone</code>. |
| In that case, you have to ask the reviewer to revisit your PR.</p> |
| <p>When your reviewer approves the PR, it will go into a queue for yet another bot called <code>@bors</code>. |
| <code>@bors</code> manages the CI build/merge queue. |
| When your PR reaches the head of the <code>@bors</code> queue, <code>@bors</code> will test out the merge by running all |
| tests against your PR on GitHub Actions. |
| This takes a lot of time to finish. |
| If all tests pass, the PR is merged and becomes part of the next nightly compiler!</p> |
| <p>There are a couple of things that may happen for some PRs during the review process</p> |
| <ul> |
| <li>If the change is substantial enough, the reviewer may request an FCP on |
| the PR. |
| This gives all members of the appropriate team a chance to review the changes.</li> |
| <li>If the change may cause breakage, the reviewer may request a <a href="./tests/crater.html">crater</a> run. |
| This compiles the compiler with your changes and then attempts to compile all |
| crates on crates.io with your modified compiler. |
| This is a great smoke test |
| to check if you introduced a change to compiler behavior that affects a large |
| portion of the ecosystem.</li> |
| <li>If the diff of your PR is large or the reviewer is busy, your PR may have |
| some merge conflicts with other PRs that happen to get merged first. |
| You should fix these merge conflicts using the normal git procedures.</li> |
| </ul> |
| <p>If you are not doing a new feature or something like that (e.g. if you are |
| fixing a bug), then that's it! |
| Thanks for your contribution :)</p> |
| <h2 id="refining-your-implementation"><a class="header" href="#refining-your-implementation">Refining your implementation</a></h2> |
| <p>As people get experience with your new feature on nightly, slight changes may |
| be proposed and unresolved questions may become resolved. |
| Updates/changes go through the same process for implementing any other changes, as described |
| above (i.e. submit a PR, go through review, wait for <code>@bors</code>, etc).</p> |
| <p>Some changes may be major enough to require an FCP and some review by Rust team members.</p> |
| <p>For the <code>?</code> macro feature, we went through a few different iterations after the |
| original implementation: <a href="https://github.com/rust-lang/rust/pull/49719">1</a>, <a href="https://github.com/rust-lang/rust/pull/51336">2</a>, <a href="https://github.com/rust-lang/rust/pull/51587">3</a>.</p> |
| <p>Along the way, we decided that <code>?</code> should not take a separator, which was |
| previously an unresolved question listed in the RFC. |
| We also changed the disambiguation strategy: we decided to remove the ability to use <code>?</code> as a |
| separator token for other repetition operators (e.g. <code>+</code> or <code>*</code>). However, |
| since this was a breaking change, we decided to do it over an edition boundary. |
| Thus, the new feature can be enabled only in edition 2018. These deviations |
| from the original RFC required <a href="https://github.com/rust-lang/rust/issues/51934">another FCP</a>.</p> |
| <h2 id="stabilization"><a class="header" href="#stabilization">Stabilization</a></h2> |
| <p>Finally, after the feature had baked for a while on nightly, a language team member |
| <a href="https://github.com/rust-lang/rust/issues/48075#issuecomment-433177613">moved to stabilize it</a>.</p> |
| <p>A <em>stabilization report</em> needs to be written that includes</p> |
| <ul> |
| <li>brief description of the behavior and any deviations from the RFC</li> |
| <li>which edition(s) are affected and how</li> |
| <li>links to a few tests to show the interesting aspects</li> |
| </ul> |
| <p>The stabilization report for our feature is <a href="https://github.com/rust-lang/rust/issues/48075#issuecomment-433243048">here</a>.</p> |
| <p>After this, <a href="https://github.com/rust-lang/rust/pull/56245">a PR is made</a> to remove the feature gate, enabling the feature by |
| default (on the 2018 edition). |
| A note is added to the <a href="https://github.com/rust-lang/rust/blob/HEAD/RELEASES.md">Release notes</a> about the feature.</p> |
| <p>Steps to stabilize the feature can be found at <a href="./stabilization_guide.html">Stabilizing Features</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><!-- date-check: Jul 2025 --> |
| <h1 id="implementing-new-language-features"><a class="header" href="#implementing-new-language-features">Implementing new language features</a></h1> |
| <p>When you want to implement a new significant feature in the compiler, |
| you need to go through this process to make sure everything goes smoothly.</p> |
| <p><strong>NOTE: This section is for <em>language</em> features, not <em>library</em> features, |
| which use <a href="./stability.html">a different process</a>.</strong></p> |
| <p>See also <a href="https://lang-team.rust-lang.org/how_to/propose.html">the Rust Language Design Team's procedures</a> for proposing changes to the language.</p> |
| <h2 id="the-rfcbot-fcp-process"><a class="header" href="#the-rfcbot-fcp-process">The @rfcbot FCP process</a></h2> |
| <p>When the change is small, uncontroversial, non-breaking, |
| and does not affect the stable language in any user-observable ways or add any new unstable features, |
| then it can be done with just writing a PR and getting an r+ from someone who knows that part of the code. |
| However, if not, more must be done. |
| Even for compiler-internal work, |
| it would be a bad idea to push a controversial change without consensus from the rest of the team |
| (both in the "distributed system" sense to make sure you don't break anything you don't know about, |
| and in the social sense to avoid PR fights).</p> |
| <p>For changes that need the consensus of a team, |
| we use the process of proposing a final comment period (FCP). |
| If you're not on the relevant team (and thus don't have @rfcbot permissions), |
| ask someone who is to start one; |
| unless they have a concern themselves, they should.</p> |
| <p>The FCP process is only needed if you need consensus – |
| if no processes require consensus for your change |
| and you don't think anyone would have a problem with it, it's OK to rely on only an r+. |
| For example, |
| it is OK to add or modify unstable command-line flags |
| or attributes in the reserved compiler-internal <code>rustc_</code> namespace |
| without an FCP for compiler development or standard library use, |
| as long as you don't expect them to be in wide use in the nightly ecosystem. |
| Some teams have lighter weight processes that they use in scenarios like this; |
| for example, |
| the compiler team recommends filing a Major Change Proposal (<a href="https://forge.rust-lang.org/compiler/proposals-and-stabilization.html#how-do-i-submit-an-mcp">MCP</a>) |
| as a lightweight way to garner support and feedback without requiring full consensus.</p> |
| <p>You don't need to have the implementation fully ready for r+ to propose an FCP, |
| but it is generally a good idea to have at least a proof of concept |
| so that people can see what you are talking about.</p> |
| <p>When an FCP is proposed, it requires all members of the team to sign off on the FCP. |
| After they all do so, |
| there's a 10-day-long "final comment period" (hence the name) where everybody can comment, |
| and if no concerns are raised, the PR/issue gets FCP approval.</p> |
| <h2 id="the-logistics-of-writing-features"><a class="header" href="#the-logistics-of-writing-features">The logistics of writing features</a></h2> |
| <p>There are a few "logistical" hoops you might need to go through |
| in order to implement a feature in a working way.</p> |
| <h3 id="warning-cycles"><a class="header" href="#warning-cycles">Warning Cycles</a></h3> |
| <p>In some cases, a feature or bugfix might break some existing programs in some edge cases. |
| In that case, |
| you'll want to do a crater run to assess the impact and possibly add a future-compatibility lint, |
| similar to those used for <a href="diagnostics.html#edition-gated-lints">edition-gated lints</a>.</p> |
| <h3 id="stability"><a class="header" href="#stability">Stability</a></h3> |
| <p>We <a href="https://github.com/rust-lang/rfcs/blob/master/text/1122-language-semver.md">value the stability of Rust</a>. |
| Code that works and runs on stable should (mostly) not break. |
| Because of that, |
| we don't want to release a feature to the world with only team consensus and code review - |
| we want to gain real-world experience on using that feature on nightly, |
| and we might want to change the feature based on that experience.</p> |
| <p>To allow for that, |
| we must make sure users don't accidentally depend on that new feature - |
| otherwise, |
| especially if experimentation takes time or is delayed and the feature takes the trains to stable, |
| it would end up de facto stable |
| and we'll not be able to make changes in it without breaking people's code.</p> |
| <p>The way we do that is that we make sure all new features are feature gated - |
| they can't be used without enabling a feature gate (<code>#[feature(foo)]</code>), |
| which can't be done in a stable/beta compiler. |
| See the <a href="implementing_new_features.html#stability-in-code">stability in code</a> section for the technical details.</p> |
| <p>Eventually, after we gain enough experience using the feature, make the necessary changes, |
| and are satisfied, we expose it to the world using the stabilization process described <a href="./stabilization_guide.html">here</a>. |
| Until then, the feature is not set in stone: |
| every part of the feature can be changed, or the feature might be completely rewritten or removed. |
| Features do not gain tenure by being unstable and unchanged for long periods of time.</p> |
| <h3 id="tracking-issues"><a class="header" href="#tracking-issues">Tracking Issues</a></h3> |
| <p>To keep track of the status of an unstable feature, |
| the experience we get while using it on nightly, |
| and of the concerns that block its stabilization, |
| every feature-gate needs a tracking issue. |
| When creating issues and PRs related to the feature, reference this tracking issue, |
| and when there are updates about the feature's progress, post those to the tracking issue.</p> |
| <p>For features that are part of an accept RFC or approved lang experiment, |
| use the tracking issue for that.</p> |
| <p>For other features, create a tracking issue for that feature. |
| The issue title should be "Tracking issue for YOUR FEATURE". |
| Use the <a href="https://github.com/rust-lang/rust/issues/new?template=tracking_issue.md">"Tracking Issue" issue template</a>.</p> |
| <h3 id="lang-experiments"><a class="header" href="#lang-experiments">Lang experiments</a></h3> |
| <p>To land in the compiler, |
| features that have user-visible effects on the language (even unstable ones) |
| must either be part of an accepted RFC or an approved <a href="https://lang-team.rust-lang.org/how_to/experiment.html">lang experiment</a>.</p> |
| <p>To propose a new lang experiment, |
| open an issue in <code>rust-lang/rust</code> that describes the motivation and the intended solution. |
| If it's accepted, this issue will become the tracking issue for the experiment, |
| so use the tracking issue <a href="https://github.com/rust-lang/rust/issues/new?template=tracking_issue.md">template</a> while also including these other details. |
| Nominate the issue for the lang team and CC <code>@rust-lang/lang</code> and <code>@rust-lang/lang-advisors</code>. |
| When the experiment is approved, the tracking issue will be marked as <code>B-experimental</code>.</p> |
| <p>Feature flags related to a lang experiment must be marked as <code>incomplete</code> |
| until an RFC is accepted for the feature.</p> |
| <h2 id="stability-in-code"><a class="header" href="#stability-in-code">Stability in code</a></h2> |
| <p>The below steps needs to be followed in order to implement a new unstable feature:</p> |
| <ol> |
| <li> |
| <p>Open or identify the <a href="implementing_new_features.html#tracking-issues">tracking issue</a>. |
| For features that are part of an accept RFC or approved lang experiment, |
| use the tracking issue for that.</p> |
| <p>Label the tracking issue with <code>C-tracking-issue</code> and the relevant <code>F-feature_name</code> label |
| (adding that label if needed).</p> |
| </li> |
| <li> |
| <p>Pick a name for the feature gate (for RFCs, use the name in the RFC).</p> |
| </li> |
| <li> |
| <p>Add the feature name to <code>rustc_span/src/symbol.rs</code> in the <code>Symbols {...}</code> block.</p> |
| <p>Note that this block must be in alphabetical order.</p> |
| </li> |
| <li> |
| <p>Add a feature gate declaration to <code>rustc_feature/src/unstable.rs</code> |
| in the unstable <code>declare_features</code> block.</p> |
| <pre><code class="language-rust ignore">/// description of feature |
| (unstable, $feature_name, "CURRENT_RUSTC_VERSION", Some($tracking_issue_number))</code></pre> |
| <p>If you haven't yet opened a tracking issue |
| (e.g. because you want initial feedback on whether the feature is likely to be accepted), |
| you can temporarily use <code>None</code> - but make sure to update it before the PR is merged!</p> |
| <p>For example:</p> |
| <pre><code class="language-rust ignore">/// Allows defining identifiers beyond ASCII. |
| (unstable, non_ascii_idents, "CURRENT_RUSTC_VERSION", Some(55467), None),</code></pre> |
| <p>Features can be marked as incomplete, |
| and trigger the warn-by-default <a href="https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#incomplete-features"><code>incomplete_features</code> lint</a> |
| by setting their type to <code>incomplete</code>:</p> |
| <pre><code class="language-rust ignore">/// Allows deref patterns. |
| (incomplete, deref_patterns, "CURRENT_RUSTC_VERSION", Some(87121), None),</code></pre> |
| <p>Feature flags related to a lang experiment must be marked as <code>incomplete</code> |
| until an RFC is accepted for the feature.</p> |
| <p>To avoid <a href="https://bors.tech/essay/2017/02/02/pitch/">semantic merge conflicts</a>, |
| use <code>CURRENT_RUSTC_VERSION</code> instead of <code>1.70</code> or another explicit version number.</p> |
| </li> |
| <li> |
| <p>Prevent usage of the new feature unless the feature gate is set. |
| You can check it in most places in the compiler |
| using the expression <code>tcx.features().$feature_name()</code>.</p> |
| <p>If the feature gate is not set, |
| you should either maintain the pre-feature behavior or raise an error, |
| depending on what makes sense. |
| Errors should generally use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/fn.feature_err.html"><code>rustc_session::parse::feature_err</code></a>. |
| For an example of adding an error, see <a href="https://github.com/rust-lang/rust/pull/81015">#81015</a>.</p> |
| <p>For features introducing new syntax, pre-expansion gating should be used instead. |
| During parsing, when the new syntax is parsed, |
| the symbol must be inserted to the current crate's <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.GatedSpans.html"><code>GatedSpans</code></a> |
| via <code>self.sess.gated_span.gate(sym::my_feature, span)</code>.</p> |
| <p>After being inserted to the gated spans, |
| the span must be checked in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast_passes/feature_gate/fn.check_crate.html"><code>rustc_ast_passes::feature_gate::check_crate</code></a> function, |
| which actually denies features. |
| Exactly how it is gated depends on the exact type of feature, |
| but most likely will use the <code>gate_all!()</code> macro.</p> |
| </li> |
| <li> |
| <p>Add a test to ensure the feature cannot be used without a feature gate, |
| by creating <code>tests/ui/feature-gates/feature-gate-$feature_name.rs</code>. |
| You can generate the corresponding <code>.stderr</code> file |
| by running <code>./x test tests/ui/feature-gates/ --bless</code>.</p> |
| </li> |
| <li> |
| <p>Add a section to the unstable book, |
| in <code>src/doc/unstable-book/src/language-features/$feature_name.md</code>.</p> |
| </li> |
| <li> |
| <p>Write a lot of tests for the new feature, preferably in <code>tests/ui/$feature_name/</code>. |
| PRs without tests will not be accepted!</p> |
| </li> |
| <li> |
| <p>Get your PR reviewed and land it. |
| You have now successfully implemented a feature in Rust!</p> |
| </li> |
| </ol> |
| <h2 id="call-for-testing"><a class="header" href="#call-for-testing">Call for testing</a></h2> |
| <p>Once the implementation is complete, |
| the feature will be available to nightly users but not yet part of stable Rust. |
| This is a good time to write a blog post on <a href="https://github.com/rust-lang/blog.rust-lang.org/">the main Rust blog</a> |
| and issue a "call for testing".</p> |
| <p>Some earlier such blog posts include:</p> |
| <ol> |
| <li><a href="https://blog.rust-lang.org/2021/08/03/GATs-stabilization-push/">The push for GATs stabilization</a></li> |
| <li><a href="https://blog.rust-lang.org/2024/09/05/impl-trait-capture-rules.html">Changes to <code>impl Trait</code> in Rust 2024</a></li> |
| <li><a href="https://blog.rust-lang.org/inside-rust/2024/08/09/async-closures-call-for-testing/">Async Closures MVP: Call for Testing!</a></li> |
| </ol> |
| <p>Alternatively, <a href="https://github.com/rust-lang/this-week-in-rust"><em>This Week in Rust</em></a> has a <a href="https://this-week-in-rust.org/blog/2025/01/22/this-week-in-rust-583/#calls-for-testing">section</a> for this. |
| One example of this having been used is:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/issues/131204#issuecomment-2569314526">Call for testing on boolean literals as cfg predicates</a></li> |
| </ul> |
| <p>Which option to choose might depend on how significant the language change is, |
| though note that the <a href="https://github.com/rust-lang/this-week-in-rust"><em>This Week in Rust</em></a> section might be less visible |
| than a dedicated post on the main Rust blog.</p> |
| <h2 id="polishing"><a class="header" href="#polishing">Polishing</a></h2> |
| <p>Giving users a polished experience means more than just implementing the feature in rustc. |
| We need to think about all of the tools and resources that we ship. |
| This work includes:</p> |
| <ul> |
| <li>Documenting the language feature in the <a href="https://github.com/rust-lang/reference">Rust Reference</a>.</li> |
| <li>Extending <a href="https://github.com/rust-lang/rustfmt"><code>rustfmt</code></a> to format any new syntax (if applicable).</li> |
| <li>Extending <a href="https://github.com/rust-lang/rust-analyzer"><code>rust-analyzer</code></a> (if applicable). |
| The extent of this work can depend on the nature of the language feature, |
| as some features don't need to be blocked on <em>full</em> support. |
| <ul> |
| <li>When a language feature degrades the user experience |
| simply by existing before support is implemented in <a href="https://github.com/rust-lang/rust-analyzer"><code>rust-analyzer</code></a>, |
| that may lead the lang team to raise a blocking concern.</li> |
| <li>Examples of such might include new syntax that <a href="https://github.com/rust-lang/rust-analyzer"><code>rust-analyzer</code></a> can't parse |
| or type inference changes it doesn't understand when those lead to bogus diagnostics.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="stabilization-1"><a class="header" href="#stabilization-1">Stabilization</a></h2> |
| <p>The final step in the feature lifecycle is <a href="./stabilization_guide.html">stabilization</a>, |
| which is when the feature becomes available to all Rust users. |
| At this point, |
| backward incompatible changes are generally no longer permitted |
| (see the lang team's <a href="https://rust-lang.github.io/rfcs/1122-language-semver.html">defined semver policies</a> for details). |
| To learn more about stabilization, see the <a href="./stabilization_guide.html">stabilization guide</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="stability-attributes"><a class="header" href="#stability-attributes">Stability attributes</a></h1> |
| <p>This section is about the stability attributes and schemes that allow stable |
| APIs to use unstable APIs internally in the rustc standard library.</p> |
| <p><strong>NOTE</strong>: this section is for <em>library</em> features, not <em>language</em> features. For instructions on |
| stabilizing a language feature see <a href="./stabilization_guide.html">Stabilizing Features</a>.</p> |
| <h2 id="unstable"><a class="header" href="#unstable">unstable</a></h2> |
| <p>The <code>#[unstable(feature = "foo", issue = "1234", reason = "lorem ipsum")]</code> |
| attribute explicitly marks an item as unstable. Items that are marked as |
| "unstable" cannot be used without a corresponding <code>#![feature]</code> attribute on |
| the crate, even on a nightly compiler. This restriction only applies across |
| crate boundaries, unstable items may be used within the crate that defines |
| them.</p> |
| <p>The <code>issue</code> field specifies the associated GitHub <a href="https://github.com/rust-lang/rust/issues">issue number</a>. This field is |
| required and all unstable features should have an associated tracking issue. In |
| rare cases where there is no sensible value <code>issue = "none"</code> is used.</p> |
| <p>The <code>unstable</code> attribute infects all sub-items, where the attribute doesn't |
| have to be reapplied. So if you apply this to a module, all items in the module |
| will be unstable.</p> |
| <p>You can make specific sub-items stable by using the <code>#[stable]</code> attribute on |
| them. The stability scheme works similarly to how <code>pub</code> works. You can have |
| public functions of nonpublic modules and you can have stable functions in |
| unstable modules or vice versa.</p> |
| <p>Previously, due to a <a href="https://github.com/rust-lang/rust/issues/15702">rustc bug</a>, stable items inside unstable modules were |
| available to stable code in that location. |
| As of <!-- date-check --> September 2024, items with <a href="https://github.com/rust-lang/rust/issues/113387">accidentally stabilized |
| paths</a> are marked with the <code>#[rustc_allowed_through_unstable_modules]</code> attribute |
| to prevent code dependent on those paths from breaking. Do <em>not</em> add this attribute |
| to any more items unless that is needed to avoid breaking changes.</p> |
| <p>The <code>unstable</code> attribute may also have the <code>soft</code> value, which makes it a |
| future-incompatible deny-by-default lint instead of a hard error. This is used |
| by the <code>bench</code> attribute which was accidentally accepted in the past. This |
| prevents breaking dependencies by leveraging Cargo's lint capping.</p> |
| <h2 id="stable"><a class="header" href="#stable">stable</a></h2> |
| <p>The <code>#[stable(feature = "foo", since = "1.420.69")]</code> attribute explicitly |
| marks an item as stabilized. Note that stable functions may use unstable things in their body.</p> |
| <h2 id="rustc_const_unstable"><a class="header" href="#rustc_const_unstable">rustc_const_unstable</a></h2> |
| <p>The <code>#[rustc_const_unstable(feature = "foo", issue = "1234", reason = "lorem ipsum")]</code> has the same interface as the <code>unstable</code> attribute. It is used to mark |
| <code>const fn</code> as having their constness be unstable. This is only needed in rare cases:</p> |
| <ul> |
| <li>If a <code>const fn</code> makes use of unstable language features or intrinsics. |
| (The compiler will tell you to add the attribute if you run into this.)</li> |
| <li>If a <code>const fn</code> is <code>#[stable]</code> but not yet intended to be const-stable.</li> |
| <li>To change the feature gate that is required to call a const-unstable intrinsic.</li> |
| </ul> |
| <p>Const-stability differs from regular stability in that it is <em>recursive</em>: a |
| <code>#[rustc_const_unstable(...)]</code> function cannot even be indirectly called from stable code. This is |
| to avoid accidentally leaking unstable compiler implementation artifacts to stable code or locking |
| us into the accidental quirks of an incomplete implementation. See the rustc_const_stable_indirect |
| and rustc_allow_const_fn_unstable attributes below for how to fine-tune this check.</p> |
| <h2 id="rustc_const_stable"><a class="header" href="#rustc_const_stable">rustc_const_stable</a></h2> |
| <p>The <code>#[rustc_const_stable(feature = "foo", since = "1.420.69")]</code> attribute explicitly marks |
| a <code>const fn</code> as having its constness be <code>stable</code>.</p> |
| <h2 id="rustc_const_stable_indirect"><a class="header" href="#rustc_const_stable_indirect">rustc_const_stable_indirect</a></h2> |
| <p>The <code>#[rustc_const_stable_indirect]</code> attribute can be added to a <code>#[rustc_const_unstable(...)]</code> |
| function to make it callable from <code>#[rustc_const_stable(...)]</code> functions. This indicates that the |
| function is ready for stable in terms of its implementation (i.e., it doesn't use any unstable |
| compiler features); the only reason it is not const-stable yet are API concerns.</p> |
| <p>This should also be added to lang items for which const-calls are synthesized in the compiler, to |
| ensure those calls do not bypass recursive const stability rules.</p> |
| <h2 id="rustc_intrinsic_const_stable_indirect"><a class="header" href="#rustc_intrinsic_const_stable_indirect">rustc_intrinsic_const_stable_indirect</a></h2> |
| <p>On an intrinsic, this attribute marks the intrinsic as "ready to be used by public stable functions". |
| If the intrinsic has a <code>rustc_const_unstable</code> attribute, it should be removed. |
| <strong>Adding this attribute to an intrinsic requires t-lang and wg-const-eval approval!</strong></p> |
| <h2 id="rustc_default_body_unstable"><a class="header" href="#rustc_default_body_unstable">rustc_default_body_unstable</a></h2> |
| <p>The <code>#[rustc_default_body_unstable(feature = "foo", issue = "1234", reason = "lorem ipsum")]</code> attribute has the same interface as the <code>unstable</code> attribute. |
| It is used to mark the default implementation for an item within a trait as |
| unstable. |
| A trait with a default-body-unstable item can be implemented stably by providing |
| an explicit body for any such item, or the default body can be used by enabling |
| its corresponding <code>#![feature]</code>.</p> |
| <h2 id="stabilizing-a-library-feature"><a class="header" href="#stabilizing-a-library-feature">Stabilizing a library feature</a></h2> |
| <p>To stabilize a feature, follow these steps:</p> |
| <ol> |
| <li>Ask a <strong>@T-libs-api</strong> member to start an FCP on the tracking issue and wait for |
| the FCP to complete (with <code>disposition-merge</code>).</li> |
| <li>Change <code>#[unstable(...)]</code> to <code>#[stable(since = "CURRENT_RUSTC_VERSION")]</code>.</li> |
| <li>Remove <code>#![feature(...)]</code> from any test or doc-test for this API. If the feature is used in the |
| compiler or tools, remove it from there as well.</li> |
| <li>If this is a <code>const fn</code>, add <code>#[rustc_const_stable(since = "CURRENT_RUSTC_VERSION")]</code>. |
| Alternatively, if this is not supposed to be const-stabilized yet, |
| add <code>#[rustc_const_unstable(...)]</code> for some new feature gate (with a new tracking issue).</li> |
| <li>Open a PR against <code>rust-lang/rust</code>. |
| <ul> |
| <li>Add the appropriate labels: <code>@rustbot modify labels: +T-libs-api</code>.</li> |
| <li>Link to the tracking issue and say "Closes #XXXXX".</li> |
| </ul> |
| </li> |
| </ol> |
| <p>You can see an example of stabilizing a feature with |
| <a href="https://github.com/rust-lang/rust/issues/81656">tracking issue #81656 with FCP</a> |
| and the associated |
| <a href="https://github.com/rust-lang/rust/pull/84642">implementation PR #84642</a>.</p> |
| <h2 id="allow_internal_unstable"><a class="header" href="#allow_internal_unstable">allow_internal_unstable</a></h2> |
| <p>Macros and compiler desugarings expose their bodies to the call |
| site. To work around not being able to use unstable things in the standard |
| library's macros, there's the <code>#[allow_internal_unstable(feature1, feature2)]</code> |
| attribute that allows the given features to be used in stable macros.</p> |
| <p>Note that if a macro is used in const context and generates a call to a |
| <code>#[rustc_const_unstable(...)]</code> function, that will <em>still</em> be rejected even with |
| <code>allow_internal_unstable</code>. Add <code>#[rustc_const_stable_indirect]</code> to the function to ensure the macro |
| cannot accidentally bypass the recursive const stability checks.</p> |
| <h2 id="rustc_allow_const_fn_unstable"><a class="header" href="#rustc_allow_const_fn_unstable">rustc_allow_const_fn_unstable</a></h2> |
| <p>As explained above, no unstable const features are allowed inside stable <code>const fn</code>, not even |
| indirectly.</p> |
| <p>However, sometimes we do know that a feature will get stabilized, just not when, or there is a |
| stable (but e.g. runtime-slow) workaround, so we could always fall back to some stable version if we |
| scrapped the unstable feature. In those cases, the <code>[rustc_allow_const_fn_unstable(feature1, feature2)]</code> attribute can be used to allow some unstable features in the body of a stable (or |
| indirectly stable) <code>const fn</code>.</p> |
| <p>You also need to take care to uphold the <code>const fn</code> invariant that calling it at runtime and |
| compile-time needs to behave the same (see also <a href="https://www.ralfj.de/blog/2018/07/19/const.html">this blog post</a>). This means that you |
| may not create a <code>const fn</code> that e.g. transmutes a memory address to an integer, |
| because the addresses of things are nondeterministic and often unknown at |
| compile-time.</p> |
| <p><strong>Always ping @rust-lang/wg-const-eval if you are adding more |
| <code>rustc_allow_const_fn_unstable</code> attributes to any <code>const fn</code>.</strong></p> |
| <h2 id="staged_api"><a class="header" href="#staged_api">staged_api</a></h2> |
| <p>Any crate that uses the <code>stable</code> or <code>unstable</code> attributes must include the |
| <code>#![feature(staged_api)]</code> attribute on the crate.</p> |
| <h2 id="deprecated"><a class="header" href="#deprecated">deprecated</a></h2> |
| <p>Deprecations in the standard library are nearly identical to deprecations in |
| user code. When <code>#[deprecated]</code> is used on an item, it must also have a <code>stable</code> |
| or <code>unstable </code>attribute.</p> |
| <p><code>deprecated</code> has the following form:</p> |
| <pre><code class="language-rust ignore">#[deprecated( |
| since = "1.38.0", |
| note = "explanation for deprecation", |
| suggestion = "other_function" |
| )]</code></pre> |
| <p>The <code>suggestion</code> field is optional. If given, it should be a string that can be |
| used as a machine-applicable suggestion to correct the warning. This is |
| typically used when the identifier is renamed, but no other significant changes |
| are necessary. When the <code>suggestion</code> field is used, you need to have |
| <code>#![feature(deprecated_suggestion)]</code> at the crate root.</p> |
| <p>Another difference from user code is that the <code>since</code> field is actually checked |
| against the current version of <code>rustc</code>. If <code>since</code> is in a future version, then |
| the <code>deprecated_in_future</code> lint is triggered which is default <code>allow</code>, but most |
| of the standard library raises it to a warning with |
| <code>#![warn(deprecated_in_future)]</code>.</p> |
| <h2 id="unstable_feature_bound"><a class="header" href="#unstable_feature_bound">unstable_feature_bound</a></h2> |
| <p>The <code>#[unstable_feature_bound(foo)]</code> attribute can be used together with <code>#[unstable]</code> attribute to mark an <code>impl</code> of stable type and stable trait as unstable. In std/core, an item annotated with <code>#[unstable_feature_bound(foo)]</code> can only be used by another item that is also annotated with <code>#[unstable_feature_bound(foo)]</code>. Outside of std/core, using an item with <code>#[unstable_feature_bound(foo)]</code> requires the feature to be enabled with <code>#![feature(foo)]</code> attribute on the crate.</p> |
| <p>Currently, the items that can be annotated with <code>#[unstable_feature_bound]</code> are:</p> |
| <ul> |
| <li><code>impl</code></li> |
| <li>free function</li> |
| <li>trait</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="request-for-stabilization"><a class="header" href="#request-for-stabilization">Request for stabilization</a></h1> |
| <p><strong>NOTE</strong>: This page is about stabilizing <em>language</em> features. |
| For stabilizing <em>library</em> features, see <a href="./stability.html#stabilizing-a-library-feature">Stabilizing a library feature</a>.</p> |
| <p>Once an unstable feature has been well-tested with no outstanding concerns, anyone may push for its stabilization, though involving the people who have worked on it is prudent. |
| Follow these steps:</p> |
| <h2 id="write-an-rfc-if-needed"><a class="header" href="#write-an-rfc-if-needed">Write an RFC, if needed</a></h2> |
| <p>If the feature was part of a <a href="https://lang-team.rust-lang.org/how_to/experiment.html">lang experiment</a>, the lang team generally will want to first accept an RFC before stabilization.</p> |
| <h2 id="documentation-prs"><a class="header" href="#documentation-prs">Documentation PRs</a></h2> |
| <p><a id="updating-documentation"></a></p> |
| <p>The feature might be documented in the <a href="https://doc.rust-lang.org/unstable-book/index.html"><code>Unstable Book</code></a>, located at <a href="https://github.com/rust-lang/rust/tree/HEAD/src/doc/unstable-book"><code>src/doc/unstable-book</code></a>. |
| Remove the page for the feature gate if it exists. |
| Integrate any useful parts of that documentation in other places.</p> |
| <p>Places that may need updated documentation include:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/reference">The Reference</a>: This must be updated, in full detail, and a member of the lang-docs team must review and approve the PR before the stabilization can be merged.</li> |
| <li><a href="https://github.com/rust-lang/book">The Book</a>: This is updated as needed. |
| If you're not sure, please open an issue on this repository and it can be discussed.</li> |
| <li>Standard library documentation: This is updated as needed. |
| Language features often don't need this, but if it's a feature that changes how idiomatic examples are written, such as when <code>?</code> was added to the language, updating these in the library documentation is important. |
| Review also the keyword documentation and ABI documentation in the standard library, as these sometimes need updates for language changes.</li> |
| <li><a href="https://github.com/rust-lang/rust-by-example">Rust by Example</a>: This is updated as needed.</li> |
| </ul> |
| <p>Prepare PRs to update documentation involving this new feature for the repositories mentioned above. |
| Maintainers of these repositories will keep these PRs open until the whole stabilization process has completed. |
| Meanwhile, we can proceed to the next step.</p> |
| <h2 id="write-a-stabilization-report"><a class="header" href="#write-a-stabilization-report">Write a stabilization report</a></h2> |
| <p>Author a stabilization report using the <a href="./stabilization_report_template.html">template found in this repository</a>.</p> |
| <p>The stabilization reports summarizes:</p> |
| <ul> |
| <li>The main design decisions and deviations since the RFC was accepted, including both decisions that were FCP'd or otherwise accepted by the language team as well as those being presented to the lang team for the first time. |
| <ul> |
| <li>Often, the final stabilized language feature has significant design deviations from the original RFC. |
| That's OK, but these deviations must be highlighted and explained carefully.</li> |
| </ul> |
| </li> |
| <li>The work that has been done since the RFC was accepted, acknowledging the main contributors that helped drive the language feature forward.</li> |
| </ul> |
| <p>The <a href="./stabilization_report_template.html"><em>Stabilization Template</em></a> includes a series of questions that aim to surface connections between this feature and lang's subteams (e.g. types, opsem, lang-docs, etc.) and to identify items that are commonly overlooked.</p> |
| <p>The stabilization report is typically posted as the main comment on the stabilization PR (see the next section).</p> |
| <h2 id="stabilization-pr"><a class="header" href="#stabilization-pr">Stabilization PR</a></h2> |
| <p>Every feature is different, and some may require steps beyond what this guide discusses.</p> |
| <p>Before the stabilization will be considered by the lang team, there must be a complete PR to the Reference describing the feature, and before the stabilization PR will be merged, this PR must have been reviewed and approved by the lang-docs team.</p> |
| <h3 id="updating-the-feature-gate-listing"><a class="header" href="#updating-the-feature-gate-listing">Updating the feature-gate listing</a></h3> |
| <p>There is a central listing of unstable feature-gates in <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_feature/src/unstable.rs"><code>compiler/rustc_feature/src/unstable.rs</code></a>. |
| Search for the <code>declare_features!</code> macro. |
| There should be an entry for the feature you are aiming to stabilize, |
| something like the following (taken from <a href="https://github.com/rust-lang/rust/issues/32409">rust-lang/rust#32409</a>:</p> |
| <pre><code class="language-rust ignore">// pub(restricted) visibilities (RFC 1422) |
| (unstable, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)),</code></pre> |
| <p>The above line should be moved to <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_feature/src/accepted.rs"><code>compiler/rustc_feature/src/accepted.rs</code></a>. |
| Entries in the <code>declare_features!</code> call are sorted, so find the correct place. |
| When it is done, it should look like:</p> |
| <pre><code class="language-rust ignore">// pub(restricted) visibilities (RFC 1422) |
| (accepted, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)), |
| // note that we changed this</code></pre> |
| <p>(Even though you will encounter version numbers in the file of past changes, you should not put the rustc version you expect your stabilization to happen in, but instead use <code>CURRENT_RUSTC_VERSION</code>.)</p> |
| <h3 id="removing-existing-uses-of-the-feature-gate"><a class="header" href="#removing-existing-uses-of-the-feature-gate">Removing existing uses of the feature-gate</a></h3> |
| <p>Next, search for the feature string (in this case, <code>pub_restricted</code>) in the codebase to find where it appears. |
| Change uses of <code>#![feature(XXX)]</code> from the <code>std</code> and any rustc crates |
| (which includes test folders under <code>library/</code> and <code>compiler/</code> but not the toplevel <code>tests/</code> one) |
| to be <code>#![cfg_attr(bootstrap, feature(XXX))]</code>. |
| This includes the feature-gate only for stage0, which is built using the current beta (this is needed because the feature is still unstable in the current beta).</p> |
| <p>Also, remove those strings from any tests (e.g. under <code>tests/</code>). If there are tests specifically targeting the feature-gate (i.e., testing that the feature-gate is required to use the feature, but nothing else), simply remove the test.</p> |
| <h3 id="do-not-require-the-feature-gate-to-use-the-feature"><a class="header" href="#do-not-require-the-feature-gate-to-use-the-feature">Do not require the feature-gate to use the feature</a></h3> |
| <p>Most importantly, remove the code which flags an error if the feature-gate is not present (since the feature is now considered stable). |
| If the feature can be detected because it employs some new syntax, then a common place for that code to be is in <code>compiler/rustc_ast_passes/src/feature_gate.rs</code>. |
| For example, you might see code like this:</p> |
| <pre><code class="language-rust ignore">gate_all!(pub_restricted, "`pub(restricted)` syntax is experimental");</code></pre> |
| <p>The <code>gate_all!</code> macro reports an error if the <code>pub_restricted</code> feature is not enabled. |
| It is not needed now that <code>pub(restricted)</code> is stable.</p> |
| <p>For more subtle features, you may find code like this:</p> |
| <pre><code class="language-rust ignore">if self.tcx.features().async_fn_in_dyn_trait() { /* XXX */ }</code></pre> |
| <p>This <code>pub_restricted</code> field (named after the feature) would ordinarily be false if the feature flag is not present and true if it is. |
| So, transform the code to assume that the field is true. |
| In this case, that would mean removing the <code>if</code> and leaving just the <code>/* XXX */</code>.</p> |
| <pre><code class="language-rust ignore">if self.tcx.sess.features.borrow().pub_restricted { /* XXX */ } |
| becomes |
| /* XXX */ |
| |
| if self.tcx.sess.features.borrow().pub_restricted && something { /* XXX */ } |
| becomes |
| if something { /* XXX */ }</code></pre> |
| <h2 id="team-nominations"><a class="header" href="#team-nominations">Team nominations</a></h2> |
| <p>When opening the stabilization PR, CC the lang team and its advisors (<code>@rust-lang/lang @rust-lang/lang-advisors</code>) and any other teams to whom the feature is relevant, e.g.:</p> |
| <ul> |
| <li><code>@rust-lang/types</code>, for type system interactions.</li> |
| <li><code>@rust-lang/opsem</code>, for interactions with unsafe code.</li> |
| <li><code>@rust-lang/compiler</code>, for implementation robustness.</li> |
| <li><code>@rust-lang/libs-api</code>, for changes to the standard library API or its guarantees.</li> |
| <li><code>@rust-lang/lang-docs</code>, for questions about how this should be documented in the Reference.</li> |
| </ul> |
| <p>After the stabilization PR is opened with the stabilization report, wait a bit for any immediate comments. |
| When such comments "simmer down" and you feel the PR is ready for consideration by the lang team, <a href="https://lang-team.rust-lang.org/how_to/nominate.html">nominate the PR</a> to get it on the agenda for consideration in an upcoming lang meeting.</p> |
| <p>If you are not a <code>rust-lang</code> organization member, you can ask your assigned reviewer to CC the relevant teams on your behalf.</p> |
| <h2 id="propose-fcp-on-the-pr"><a class="header" href="#propose-fcp-on-the-pr">Propose FCP on the PR</a></h2> |
| <p>After the lang team and other relevant teams review the stabilization, and after you have answered any questions they may have had, a member of one of the teams may propose to accept the stabilization by commenting:</p> |
| <pre><code class="language-text">@rfcbot fcp merge |
| </code></pre> |
| <p>Once enough team members have reviewed, the PR will move into a "final comment period" (FCP). |
| If no new concerns are raised, this period will complete and the PR can be merged after implementation review in the usual way.</p> |
| <h2 id="reviewing-and-merging-stabilizations"><a class="header" href="#reviewing-and-merging-stabilizations">Reviewing and merging stabilizations</a></h2> |
| <p>On a stabilization, before giving it the <code>r+</code>, ensure that the PR:</p> |
| <ul> |
| <li>Matches what the team proposed for stabilization and what is documented in the Reference PR.</li> |
| <li>Includes any changes the team decided to request along the way in order to resolve or avoid concerns.</li> |
| <li>Is otherwise exactly what is described in the stabilization report and in any relevant RFCs or prior lang FCPs.</li> |
| <li>Does not expose on stable behaviors other than those specified, accepted for stabilization, and documented in the Reference.</li> |
| <li>Has sufficient tests to convincingly demonstrate these things.</li> |
| <li>Is accompanied by a PR to the Reference than has been reviewed and approved by a member of lang-docs.</li> |
| </ul> |
| <p>In particular, when reviewing the PR, keep an eye out for any user-visible details that the lang team failed to consider and specify. |
| If you find one, describe it and nominate the PR for the lang team.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="stabilization-report-template"><a class="header" href="#stabilization-report-template">Stabilization report template</a></h1> |
| <h2 id="what-is-this"><a class="header" href="#what-is-this">What is this?</a></h2> |
| <p>This is a template for <a href="./stabilization_guide.html">stabilization reports</a> of <strong>language features</strong>. The questions aim to solicit the details most often needed. These details help reviewers to identify potential problems upfront. Not all parts of the template will apply to every stabilization. If a question doesn't apply, explain briefly why.</p> |
| <p>Copy everything after the separator and edit it as Markdown. Replace each <em>TODO</em> with your answer.</p> |
| <hr /> |
| <h1 id="stabilization-report"><a class="header" href="#stabilization-report">Stabilization report</a></h1> |
| <h2 id="summary"><a class="header" href="#summary">Summary</a></h2> |
| <blockquote> |
| <p>Remind us what this feature is and what value it provides. Tell the story of what led up to this stabilization.</p> |
| <p>E.g., see:</p> |
| <ul> |
| <li><a href="https://web.archive.org/web/20250329190642/https://github.com/rust-lang/rust/pull/115822">Stabilize AFIT/RPITIT</a></li> |
| <li><a href="https://web.archive.org/web/20250321214601/https://github.com/rust-lang/rust/pull/138424">Stabilize RTN</a></li> |
| <li><a href="https://web.archive.org/web/20250124214256/https://github.com/rust-lang/rust/pull/120700">Stabilize ATPIT</a></li> |
| <li><a href="https://web.archive.org/web/20250312173538/https://github.com/rust-lang/rust/pull/127672">Stabilize opaque type precise capturing</a></li> |
| </ul> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <p>Tracking:</p> |
| <ul> |
| <li><em>TODO</em> (Link to tracking issue.)</li> |
| </ul> |
| <p>Reference PRs:</p> |
| <ul> |
| <li><em>TODO</em> (Link to Reference PRs.)</li> |
| </ul> |
| <p>cc @rust-lang/lang @rust-lang/lang-advisors</p> |
| <h3 id="what-is-stabilized"><a class="header" href="#what-is-stabilized">What is stabilized</a></h3> |
| <blockquote> |
| <p>Describe each behavior being stabilized and give a short example of code that will now be accepted.</p> |
| </blockquote> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>todo!() |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="what-isnt-stabilized"><a class="header" href="#what-isnt-stabilized">What isn't stabilized</a></h3> |
| <blockquote> |
| <p>Describe any parts of the feature not being stabilized. Talk about what we might want to do later and what doors are being left open for that. If what we're not stabilizing might lead to surprises for users, talk about that in particular.</p> |
| </blockquote> |
| <h2 id="design"><a class="header" href="#design">Design</a></h2> |
| <h3 id="reference"><a class="header" href="#reference">Reference</a></h3> |
| <blockquote> |
| <p>What updates are needed to the Reference? Link to each PR. If the Reference is missing content needed for describing this feature, discuss that.</p> |
| </blockquote> |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| <h3 id="rfc-history"><a class="header" href="#rfc-history">RFC history</a></h3> |
| <blockquote> |
| <p>What RFCs have been accepted for this feature?</p> |
| </blockquote> |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| <h3 id="answers-to-unresolved-questions"><a class="header" href="#answers-to-unresolved-questions">Answers to unresolved questions</a></h3> |
| <blockquote> |
| <p>What questions were left unresolved by the RFC? How have they been answered? Link to any relevant lang decisions.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="post-rfc-changes"><a class="header" href="#post-rfc-changes">Post-RFC changes</a></h3> |
| <blockquote> |
| <p>What other user-visible changes have occurred since the RFC was accepted? Describe both changes that the lang team accepted (and link to those decisions) as well as changes that are being presented to the team for the first time in this stabilization report.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="key-points"><a class="header" href="#key-points">Key points</a></h3> |
| <blockquote> |
| <p>What decisions have been most difficult and what behaviors to be stabilized have proved most contentious? Summarize the major arguments on all sides and link to earlier documents and discussions.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="nightly-extensions"><a class="header" href="#nightly-extensions">Nightly extensions</a></h3> |
| <blockquote> |
| <p>Are there extensions to this feature that remain unstable? How do we know that we are not accidentally committing to those?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="doors-closed"><a class="header" href="#doors-closed">Doors closed</a></h3> |
| <blockquote> |
| <p>What doors does this stabilization close for later changes to the language? E.g., does this stabilization make any other RFCs, lang experiments, or known in-flight proposals more difficult or impossible to do later?</p> |
| </blockquote> |
| <h2 id="feedback"><a class="header" href="#feedback">Feedback</a></h2> |
| <h3 id="call-for-testing-1"><a class="header" href="#call-for-testing-1">Call for testing</a></h3> |
| <blockquote> |
| <p>Has a "call for testing" been done? If so, what feedback was received?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="nightly-use"><a class="header" href="#nightly-use">Nightly use</a></h3> |
| <blockquote> |
| <p>Do any known nightly users use this feature? Counting instances of <code>#![feature(FEATURE_NAME)]</code> on GitHub with grep might be informative.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h2 id="implementation-1"><a class="header" href="#implementation-1">Implementation</a></h2> |
| <h3 id="major-parts"><a class="header" href="#major-parts">Major parts</a></h3> |
| <blockquote> |
| <p>Summarize the major parts of the implementation and provide links into the code and to relevant PRs.</p> |
| <p>See, e.g., this breakdown of the major parts of async closures:</p> |
| <ul> |
| <li><a href="https://rustc-dev-guide.rust-lang.org/coroutine-closures.html">https://rustc-dev-guide.rust-lang.org/coroutine-closures.html</a></li> |
| </ul> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="coverage"><a class="header" href="#coverage">Coverage</a></h3> |
| <blockquote> |
| <p>Summarize the test coverage of this feature.</p> |
| <p>Consider what the "edges" of this feature are. We're particularly interested in seeing tests that assure us about exactly what nearby things we're not stabilizing. Tests should of course comprehensively demonstrate that the feature works. Think too about demonstrating the diagnostics seen when common mistakes are made and the feature is used incorrectly.</p> |
| <p>Within each test, include a comment at the top describing the purpose of the test and what set of invariants it intends to demonstrate. This is a great help to our review.</p> |
| <p>Describe any known or intentional gaps in test coverage.</p> |
| <p>Contextualize and link to test folders and individual tests.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="outstanding-bugs"><a class="header" href="#outstanding-bugs">Outstanding bugs</a></h3> |
| <blockquote> |
| <p>What outstanding bugs involve this feature? List them. Should any block the stabilization? Discuss why or why not.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <ul> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| </ul> |
| <h3 id="outstanding-fixmes"><a class="header" href="#outstanding-fixmes">Outstanding FIXMEs</a></h3> |
| <blockquote> |
| <p>What FIXMEs are still in the code for that feature and why is it OK to leave them there?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="tool-changes"><a class="header" href="#tool-changes">Tool changes</a></h3> |
| <blockquote> |
| <p>What changes must be made to our other tools to support this feature. Has this work been done? Link to any relevant PRs and issues.</p> |
| </blockquote> |
| <ul> |
| <li><input disabled="" type="checkbox"/> |
| rustfmt |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| <li><input disabled="" type="checkbox"/> |
| rust-analyzer |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| <li><input disabled="" type="checkbox"/> |
| rustdoc (both JSON and HTML) |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| <li><input disabled="" type="checkbox"/> |
| cargo |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| <li><input disabled="" type="checkbox"/> |
| clippy |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| <li><input disabled="" type="checkbox"/> |
| rustup |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| <li><input disabled="" type="checkbox"/> |
| docs.rs |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| </li> |
| </ul> |
| <p><em>TODO</em></p> |
| <h3 id="breaking-changes-1"><a class="header" href="#breaking-changes-1">Breaking changes</a></h3> |
| <blockquote> |
| <p>If this stabilization represents a known breaking change, link to the crater report, the analysis of the crater report, and to all PRs we've made to ecosystem projects affected by this breakage. Discuss any limitations of what we're able to know about or to fix.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <p>Crater report:</p> |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| <p>Crater analysis:</p> |
| <ul> |
| <li><em>TODO</em></li> |
| </ul> |
| <p>PRs to affected crates:</p> |
| <ul> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| </ul> |
| <h2 id="type-system-opsem"><a class="header" href="#type-system-opsem">Type system, opsem</a></h2> |
| <h3 id="compile-time-checks"><a class="header" href="#compile-time-checks">Compile-time checks</a></h3> |
| <blockquote> |
| <p>What compilation-time checks are done that are needed to prevent undefined behavior?</p> |
| <p>Link to tests demonstrating that these checks are being done.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <ul> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| </ul> |
| <h3 id="type-system-rules"><a class="header" href="#type-system-rules">Type system rules</a></h3> |
| <blockquote> |
| <p>What type system rules are enforced for this feature and what is the purpose of each?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="sound-by-default"><a class="header" href="#sound-by-default">Sound by default?</a></h3> |
| <blockquote> |
| <p>Does the feature's implementation need specific checks to prevent UB, or is it sound by default and need specific opt-in to perform the dangerous/unsafe operations? If it is not sound by default, what is the rationale?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="breaks-the-am"><a class="header" href="#breaks-the-am">Breaks the AM?</a></h3> |
| <blockquote> |
| <p>Can users use this feature to introduce undefined behavior, or use this feature to break the abstraction of Rust and expose the underlying assembly-level implementation? Describe this if so.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h2 id="common-interactions"><a class="header" href="#common-interactions">Common interactions</a></h2> |
| <h3 id="temporaries"><a class="header" href="#temporaries">Temporaries</a></h3> |
| <blockquote> |
| <p>Does this feature introduce new expressions that can produce temporaries? What are the scopes of those temporaries?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="drop-order"><a class="header" href="#drop-order">Drop order</a></h3> |
| <blockquote> |
| <p>Does this feature raise questions about the order in which we should drop values? Talk about the decisions made here and how they're consistent with our earlier decisions.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="pre-expansion--post-expansion"><a class="header" href="#pre-expansion--post-expansion">Pre-expansion / post-expansion</a></h3> |
| <blockquote> |
| <p>Does this feature raise questions about what should be accepted pre-expansion (e.g. in code covered by <code>#[cfg(false)]</code>) versus what should be accepted post-expansion? What decisions were made about this?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="edition-hygiene"><a class="header" href="#edition-hygiene">Edition hygiene</a></h3> |
| <blockquote> |
| <p>If this feature is gated on an edition, how do we decide, in the context of the edition hygiene of tokens, whether to accept or reject code. E.g., what token do we use to decide?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="semver-implications"><a class="header" href="#semver-implications">SemVer implications</a></h3> |
| <blockquote> |
| <p>Does this feature create any new ways in which library authors must take care to prevent breaking downstreams when making minor-version releases? Describe these. Are these new hazards "major" or "minor" according to <a href="https://rust-lang.github.io/rfcs/1105-api-evolution.html">RFC 1105</a>?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h3 id="exposing-other-features"><a class="header" href="#exposing-other-features">Exposing other features</a></h3> |
| <blockquote> |
| <p>Are there any other unstable features whose behavior may be exposed by this feature in any way? What features present the highest risk of that?</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h2 id="history"><a class="header" href="#history">History</a></h2> |
| <blockquote> |
| <p>List issues and PRs that are important for understanding how we got here.</p> |
| </blockquote> |
| <ul> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| <li><em>TODO</em></li> |
| </ul> |
| <h2 id="acknowledgments"><a class="header" href="#acknowledgments">Acknowledgments</a></h2> |
| <blockquote> |
| <p>Summarize contributors to the feature by name for recognition and so that those people are notified about the stabilization. Does anyone who worked on this <em>not</em> think it should be stabilized right now? We'd like to hear about that if so.</p> |
| </blockquote> |
| <p><em>TODO</em></p> |
| <h2 id="open-items"><a class="header" href="#open-items">Open items</a></h2> |
| <blockquote> |
| <p>List any known items that have not yet been completed and that should be before this is stabilized.</p> |
| </blockquote> |
| <ul> |
| <li><input disabled="" type="checkbox"/> |
| <em>TODO</em></li> |
| <li><input disabled="" type="checkbox"/> |
| <em>TODO</em></li> |
| <li><input disabled="" type="checkbox"/> |
| <em>TODO</em></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="feature-gates"><a class="header" href="#feature-gates">Feature gates</a></h1> |
| <p>This chapter is intended to provide basic help for adding, removing, and |
| modifying feature gates.</p> |
| <p>Note that this is specific to <em>language</em> feature gates; <em>library</em> feature gates use <a href="./stability.html">a different |
| mechanism</a>.</p> |
| <h2 id="adding-a-feature-gate"><a class="header" href="#adding-a-feature-gate">Adding a feature gate</a></h2> |
| <p>See <a href="./implementing_new_features.html#stability-in-code">"Stability in code"</a> in the "Implementing new features" section for instructions.</p> |
| <h2 id="removing-a-feature-gate"><a class="header" href="#removing-a-feature-gate">Removing a feature gate</a></h2> |
| <p>To remove a feature gate, follow these steps:</p> |
| <ol> |
| <li> |
| <p>Remove the feature gate declaration in <code>rustc_feature/src/unstable.rs</code>. |
| It will look like this:</p> |
| <pre><code class="language-rust ignore">/// description of feature |
| (unstable, $feature_name, "$version", Some($tracking_issue_number))</code></pre> |
| </li> |
| <li> |
| <p>Add a modified version of the feature gate declaration that you just |
| removed to <code>rustc_feature/src/removed.rs</code>:</p> |
| <pre><code class="language-rust ignore">/// description of feature |
| (removed, $old_feature_name, "$version", Some($tracking_issue_number), |
| Some("$why_it_was_removed"))</code></pre> |
| </li> |
| </ol> |
| <h2 id="renaming-a-feature-gate"><a class="header" href="#renaming-a-feature-gate">Renaming a feature gate</a></h2> |
| <p>To rename a feature gate, follow these steps (the first two are the same steps |
| to follow when <a href="feature-gates.html#removing-a-feature-gate">removing a feature gate</a>):</p> |
| <ol> |
| <li> |
| <p>Remove the old feature gate declaration in <code>rustc_feature/src/unstable.rs</code>. |
| It will look like this:</p> |
| <pre><code class="language-rust ignore">/// description of feature |
| (unstable, $old_feature_name, "$version", Some($tracking_issue_number))</code></pre> |
| </li> |
| <li> |
| <p>Add a modified version of the old feature gate declaration that you just |
| removed to <code>rustc_feature/src/removed.rs</code>:</p> |
| <pre><code class="language-rust ignore">/// description of feature |
| /// Renamed to `$new_feature_name` |
| (removed, $old_feature_name, "$version", Some($tracking_issue_number), |
| Some("renamed to `$new_feature_name`"))</code></pre> |
| </li> |
| <li> |
| <p>Add a feature gate declaration with the new name to |
| <code>rustc_feature/src/unstable.rs</code>. It should look very similar to the old |
| declaration:</p> |
| <pre><code class="language-rust ignore">/// description of feature |
| (unstable, $new_feature_name, "$version", Some($tracking_issue_number))</code></pre> |
| </li> |
| </ol> |
| <h2 id="stabilizing-a-feature"><a class="header" href="#stabilizing-a-feature">Stabilizing a feature</a></h2> |
| <p>See <a href="./stabilization_guide.html#updating-the-feature-gate-listing">"Updating the feature-gate listing"</a> in the "Stabilizing Features" chapter |
| for instructions. There are additional steps you will need to take beyond just |
| updating the declaration!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="coding-conventions"><a class="header" href="#coding-conventions">Coding conventions</a></h1> |
| <p>This file offers some tips on the coding conventions for rustc. This |
| chapter covers <a href="conventions.html#formatting">formatting</a>, <a href="conventions.html#cc">coding for correctness</a>, |
| <a href="conventions.html#cio">using crates from crates.io</a>, and some tips on |
| <a href="conventions.html#er">structuring your PR for easy review</a>.</p> |
| <p><a id="formatting"></a></p> |
| <h2 id="formatting-and-the-tidy-script"><a class="header" href="#formatting-and-the-tidy-script">Formatting and the tidy script</a></h2> |
| <p>rustc is moving towards the <a href="https://github.com/rust-dev-tools/fmt-rfcs">Rust standard coding style</a>.</p> |
| <p>However, for now we don't use stable <code>rustfmt</code>; we use a pinned version with a |
| special config, so this may result in different style from normal <a href="https://github.com/rust-lang/rustfmt"><code>rustfmt</code></a>. |
| Therefore, formatting this repository using <code>cargo fmt</code> is not recommended.</p> |
| <p>Instead, formatting should be done using <code>./x fmt</code>. It's a good habit to run |
| <code>./x fmt</code> before every commit, as this reduces conflicts later.</p> |
| <p>Formatting is checked by the <code>tidy</code> script. It runs automatically when you do |
| <code>./x test</code> and can be run in isolation with <code>./x fmt --check</code>.</p> |
| <p>If you want to use format-on-save in your editor, the pinned version of |
| <code>rustfmt</code> is built under <code>build/<target>/stage0/bin/rustfmt</code>.</p> |
| <h3 id="formatting-c-code"><a class="header" href="#formatting-c-code">Formatting C++ code</a></h3> |
| <p>The compiler contains some C++ code for interfacing with parts of LLVM that |
| don't have a stable C API. |
| When modifying that code, use this command to format it:</p> |
| <pre><code class="language-console">./x test tidy --extra-checks cpp:fmt --bless |
| </code></pre> |
| <p>This uses a pinned version of <code>clang-format</code>, to avoid relying on the local |
| environment.</p> |
| <h3 id="formatting-and-linting-python-code"><a class="header" href="#formatting-and-linting-python-code">Formatting and linting Python code</a></h3> |
| <p>The Rust repository contains quite a lot of Python code. We try to keep |
| it both linted and formatted by the <a href="https://github.com/astral-sh/ruff">ruff</a> tool.</p> |
| <p>When modifying Python code, use this command to format it:</p> |
| <pre><code class="language-console">./x test tidy --extra-checks py:fmt --bless |
| </code></pre> |
| <p>And, the following command to run lints:</p> |
| <pre><code class="language-console">./x test tidy --extra-checks py:lint |
| </code></pre> |
| <p>These use a pinned version of <code>ruff</code>, to avoid relying on the local environment.</p> |
| <p><a id="copyright"></a></p> |
| <!-- REUSE-IgnoreStart --> |
| <!-- Prevent REUSE from interpreting the heading as a copyright notice --> |
| <h3 id="copyright-notice"><a class="header" href="#copyright-notice">Copyright notice</a></h3> |
| <!-- REUSE-IgnoreEnd --> |
| <p>In the past, files began with a copyright and license notice. Please <strong>omit</strong> |
| this notice for new files licensed under the standard terms (dual |
| MIT/Apache-2.0).</p> |
| <p>All of the copyright notices should be gone by now, but if you come across one |
| in the rust-lang/rust repo, feel free to open a PR to remove it.</p> |
| <h3 id="line-length"><a class="header" href="#line-length">Line length</a></h3> |
| <p>Lines should be at most 100 characters. It's even better if you can |
| keep things to 80.</p> |
| <p>Sometimes, and particularly for tests, it can be necessary to exempt yourself from this limit. |
| In that case, you can add a comment towards the top of the file like so:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// ignore-tidy-linelength |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="tabs-vs-spaces"><a class="header" href="#tabs-vs-spaces">Tabs vs spaces</a></h3> |
| <p>Prefer 4-space indents.</p> |
| <p><a id="cc"></a></p> |
| <h2 id="coding-for-correctness"><a class="header" href="#coding-for-correctness">Coding for correctness</a></h2> |
| <p>Beyond formatting, there are a few other tips that are worth |
| following.</p> |
| <h3 id="prefer-exhaustive-matches"><a class="header" href="#prefer-exhaustive-matches">Prefer exhaustive matches</a></h3> |
| <p>Using <code>_</code> in a match is convenient, but it means that when new |
| variants are added to the enum, they may not get handled correctly. |
| Ask yourself: if a new variant were added to this enum, what's the |
| chance that it would want to use the <code>_</code> code, versus having some |
| other treatment? Unless the answer is "low", then prefer an |
| exhaustive match.</p> |
| <p>The same advice applies to <code>if let</code> and <code>while let</code>, |
| which are effectively tests for a single variant.</p> |
| <h3 id="use-todo-comments-for-things-you-dont-want-to-forget"><a class="header" href="#use-todo-comments-for-things-you-dont-want-to-forget">Use "TODO" comments for things you don't want to forget</a></h3> |
| <p>As a useful tool to yourself, you can insert a <code>// TODO</code> comment |
| for something that you want to get back to before you land your PR:</p> |
| <pre><code class="language-rust ignore">fn do_something() { |
| if something_else { |
| unimplemented!(); // TODO write this |
| } |
| }</code></pre> |
| <p>The tidy script will report an error for a <code>// TODO</code> comment, so this |
| code would not be able to land until the TODO is fixed (or removed).</p> |
| <p>This can also be useful in a PR as a way to signal from one commit that you are |
| leaving a bug that a later commit will fix:</p> |
| <pre><code class="language-rust ignore">if foo { |
| return true; // TODO wrong, but will be fixed in a later commit |
| }</code></pre> |
| <p><a id="cio"></a></p> |
| <h2 id="using-crates-from-cratesio"><a class="header" href="#using-crates-from-cratesio">Using crates from crates.io</a></h2> |
| <p>See the <a href="./crates-io.html">crates.io dependencies</a> section.</p> |
| <p><a id="er"></a></p> |
| <h2 id="how-to-structure-your-pr"><a class="header" href="#how-to-structure-your-pr">How to structure your PR</a></h2> |
| <p>How you prepare the commits in your PR can make a big difference for the |
| reviewer. Here are some tips.</p> |
| <p><strong>Isolate "pure refactorings" into their own commit.</strong> For example, if |
| you rename a method, then put that rename into its own commit, along |
| with the renames of all the uses.</p> |
| <p><strong>More commits is usually better.</strong> If you are doing a large change, |
| it's almost always better to break it up into smaller steps that can |
| be independently understood. The one thing to be aware of is that if |
| you introduce some code following one strategy, then change it |
| dramatically (versus adding to it) in a later commit, that |
| 'back-and-forth' can be confusing.</p> |
| <p><strong>Format liberally.</strong> While only the final commit of a PR must be correctly |
| formatted, it is both easier to review and less noisy to format each commit |
| individually using <code>./x fmt</code>.</p> |
| <p><strong>No merges.</strong> We do not allow merge commits into our history, other |
| than those by bors. If you get a merge conflict, rebase instead via a |
| command like <code>git rebase --interactive rust-lang/main</code> (presuming you use the |
| name <code>rust-lang</code> for your remote).</p> |
| <p><strong>Individual commits do not have to build (but it's nice).</strong> We do not |
| require that every intermediate commit successfully builds – we only |
| expect to be able to bisect at a PR level. However, if you <em>can</em> make |
| individual commits build, that is always helpful.</p> |
| <h2 id="naming-conventions"><a class="header" href="#naming-conventions">Naming conventions</a></h2> |
| <p>Apart from normal Rust style/naming conventions, there are also some specific |
| to the compiler.</p> |
| <ul> |
| <li> |
| <p><code>cx</code> tends to be short for "context" and is often used as a suffix. For |
| example, <code>tcx</code> is a common name for the <a href="./ty.html">Typing Context</a>.</p> |
| </li> |
| <li> |
| <p><a href="./ty.html"><code>'tcx</code></a> is used as the lifetime name for the Typing Context.</p> |
| </li> |
| <li> |
| <p>Because <code>crate</code> is a keyword, if you need a variable to represent something |
| crate-related, often the spelling is changed to <code>krate</code>.</p> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="procedures-for-breaking-changes"><a class="header" href="#procedures-for-breaking-changes">Procedures for breaking changes</a></h1> |
| <p>This page defines the best practices procedure for making bug fixes or soundness |
| corrections in the compiler that can cause existing code to stop compiling. This |
| text is based on |
| <a href="https://github.com/rust-lang/rfcs/blob/master/text/1589-rustc-bug-fix-procedure.md">RFC 1589</a>.</p> |
| <h1 id="motivation"><a class="header" href="#motivation">Motivation</a></h1> |
| <p>From time to time, we encounter the need to make a bug fix, soundness |
| correction, or other change in the compiler which will cause existing code to |
| stop compiling. When this happens, it is important that we handle the change in |
| a way that gives users of Rust a smooth transition. What we want to avoid is |
| that existing programs suddenly stop compiling with opaque error messages: we |
| would prefer to have a gradual period of warnings, with clear guidance as to |
| what the problem is, how to fix it, and why the change was made. This RFC |
| describes the procedure that we have been developing for handling breaking |
| changes that aims to achieve that kind of smooth transition.</p> |
| <p>One of the key points of this policy is that (a) warnings should be issued |
| initially rather than hard errors if at all possible and (b) every change that |
| causes existing code to stop compiling will have an associated tracking issue. |
| This issue provides a point to collect feedback on the results of that change. |
| Sometimes changes have unexpectedly large consequences or there may be a way to |
| avoid the change that was not considered. In those cases, we may decide to |
| change course and roll back the change, or find another solution (if warnings |
| are being used, this is particularly easy to do).</p> |
| <h3 id="what-qualifies-as-a-bug-fix"><a class="header" href="#what-qualifies-as-a-bug-fix">What qualifies as a bug fix?</a></h3> |
| <p>Note that this RFC does not try to define when a breaking change is permitted. |
| That is already covered under <a href="https://github.com/rust-lang/rfcs/blob/master/text/1122-language-semver.md">RFC 1122</a>. This document assumes that the |
| change being made is in accordance with those policies. Here is a summary of the |
| conditions from RFC 1122:</p> |
| <ul> |
| <li><strong>Soundness changes:</strong> Fixes to holes uncovered in the type system.</li> |
| <li><strong>Compiler bugs:</strong> Places where the compiler is not implementing the specified |
| semantics found in an RFC or lang-team decision.</li> |
| <li><strong>Underspecified language semantics:</strong> Clarifications to grey areas where the |
| compiler behaves inconsistently and no formal behavior had been previously |
| decided.</li> |
| </ul> |
| <p>Please see <a href="https://github.com/rust-lang/rfcs/blob/master/text/1122-language-semver.md">the RFC</a> for full details!</p> |
| <h1 id="detailed-design"><a class="header" href="#detailed-design">Detailed design</a></h1> |
| <p>The procedure for making a breaking change is as follows (each of these steps is |
| described in more detail below):</p> |
| <ol> |
| <li>Do a <strong>crater run</strong> to assess the impact of the change.</li> |
| <li>Make a <strong>special tracking issue</strong> dedicated to the change.</li> |
| <li>Do not report an error right away. Instead, <strong>issue forwards-compatibility |
| lint warnings</strong>. |
| <ul> |
| <li>Sometimes this is not straightforward. See the text below for suggestions |
| on different techniques we have employed in the past.</li> |
| <li>For cases where warnings are infeasible: |
| <ul> |
| <li>Report errors, but make every effort to give a targeted error message |
| that directs users to the tracking issue</li> |
| <li>Submit PRs to all known affected crates that fix the issue |
| <ul> |
| <li>or, at minimum, alert the owners of those crates to the problem and |
| direct them to the tracking issue</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>Once the change has been in the wild for at least one cycle, we can |
| <strong>stabilize the change</strong>, converting those warnings into errors.</li> |
| </ol> |
| <p>Finally, for changes to <code>rustc_ast</code> that will affect plugins, the general policy |
| is to batch these changes. That is discussed below in more detail.</p> |
| <h3 id="tracking-issue"><a class="header" href="#tracking-issue">Tracking issue</a></h3> |
| <p>Every breaking change should be accompanied by a <strong>dedicated tracking issue</strong> |
| for that change. The main text of this issue should describe the change being |
| made, with a focus on what users must do to fix their code. The issue should be |
| approachable and practical; it may make sense to direct users to an RFC or some |
| other issue for the full details. The issue also serves as a place where users |
| can comment with questions or other concerns.</p> |
| <p>A template for these breaking-change tracking issues can be found |
| <a href="https://github.com/rust-lang/rust/issues/new?template=tracking_issue_future.md">here</a>. An example of how such an issue should look can be <a href="https://gist.github.com/nikomatsakis/631ec8b4af9a18b5d062d9d9b7d3d967">found |
| here</a>.</p> |
| <h3 id="issuing-future-compatibility-warnings"><a class="header" href="#issuing-future-compatibility-warnings">Issuing future compatibility warnings</a></h3> |
| <p>The best way to handle a breaking change is to begin by issuing |
| future-compatibility warnings. These are a special category of lint warning. |
| Adding a new future-compatibility warning can be done as follows.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// 1. Define the lint in `compiler/rustc_middle/src/lint/builtin.rs`: |
| declare_lint! { |
| pub YOUR_ERROR_HERE, |
| Warn, |
| "illegal use of foo bar baz" |
| } |
| |
| // 2. Add to the list of HardwiredLints in the same file: |
| impl LintPass for HardwiredLints { |
| fn get_lints(&self) -> LintArray { |
| lint_array!( |
| .., |
| YOUR_ERROR_HERE |
| ) |
| } |
| } |
| |
| // 3. Register the lint in `compiler/rustc_lint/src/lib.rs`: |
| store.register_future_incompatible(sess, vec![ |
| ..., |
| FutureIncompatibleInfo { |
| id: LintId::of(YOUR_ERROR_HERE), |
| reference: "issue #1234", // your tracking issue here! |
| }, |
| ]); |
| |
| // 4. Report the lint: |
| tcx.lint_node( |
| lint::builtin::YOUR_ERROR_HERE, |
| path_id, |
| binding.span, |
| format!("some helper message here")); |
| <span class="boring">}</span></code></pre></pre> |
| <h4 id="helpful-techniques"><a class="header" href="#helpful-techniques">Helpful techniques</a></h4> |
| <p>It can often be challenging to filter out new warnings from older, pre-existing |
| errors. One technique that has been used in the past is to run the older code |
| unchanged and collect the errors it would have reported. You can then issue |
| warnings for any errors you would give which do not appear in that original set. |
| Another option is to abort compilation after the original code completes if |
| errors are reported: then you know that your new code will only execute when |
| there were no errors before.</p> |
| <h4 id="crater-and-cratesio"><a class="header" href="#crater-and-cratesio">Crater and crates.io</a></h4> |
| <p><a href="./tests/crater.html">Crater</a> is a bot that will compile all crates.io crates and many |
| public github repos with the compiler with your changes. A report will then be |
| generated with crates that ceased to compile with or began to compile with your |
| changes. Crater runs can take a few days to complete.</p> |
| <p>We should always do a crater run to assess impact. It is polite and considerate |
| to at least notify the authors of affected crates the breaking change. If we can |
| submit PRs to fix the problem, so much the better.</p> |
| <h4 id="is-it-ever-acceptable-to-go-directly-to-issuing-errors"><a class="header" href="#is-it-ever-acceptable-to-go-directly-to-issuing-errors">Is it ever acceptable to go directly to issuing errors?</a></h4> |
| <p>Changes that are believed to have negligible impact can go directly to issuing |
| an error. One rule of thumb would be to check against <code>crates.io</code>: if fewer than |
| 10 <strong>total</strong> affected projects are found (<strong>not</strong> root errors), we can move |
| straight to an error. In such cases, we should still make the "breaking change" |
| page as before, and we should ensure that the error directs users to this page. |
| In other words, everything should be the same except that users are getting an |
| error, and not a warning. Moreover, we should submit PRs to the affected |
| projects (ideally before the PR implementing the change lands in rustc).</p> |
| <p>If the impact is not believed to be negligible (e.g., more than 10 crates are |
| affected), then warnings are required (unless the compiler team agrees to grant |
| a special exemption in some particular case). If implementing warnings is not |
| feasible, then we should make an aggressive strategy of migrating crates before |
| we land the change so as to lower the number of affected crates. Here are some |
| techniques for approaching this scenario:</p> |
| <ol> |
| <li>Issue warnings for subparts of the problem, and reserve the new errors for |
| the smallest set of cases you can.</li> |
| <li>Try to give a very precise error message that suggests how to fix the problem |
| and directs users to the tracking issue.</li> |
| <li>It may also make sense to layer the fix: |
| <ul> |
| <li>First, add warnings where possible and let those land before proceeding to |
| issue errors.</li> |
| <li>Work with authors of affected crates to ensure that corrected versions are |
| available <em>before</em> the fix lands, so that downstream users can use them.</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 id="stabilization-2"><a class="header" href="#stabilization-2">Stabilization</a></h3> |
| <p>After a change is made, we will <strong>stabilize</strong> the change using the same process |
| that we use for unstable features:</p> |
| <ul> |
| <li> |
| <p>After a new release is made, we will go through the outstanding tracking |
| issues corresponding to breaking changes and nominate some of them for <strong>final |
| comment period</strong> (FCP).</p> |
| </li> |
| <li> |
| <p>The FCP for such issues lasts for one cycle. In the final week or two of the |
| cycle, we will review comments and make a final determination:</p> |
| <ul> |
| <li>Convert to error: the change should be made into a hard error.</li> |
| <li>Revert: we should remove the warning and continue to allow the older code to |
| compile.</li> |
| <li>Defer: can't decide yet, wait longer, or try other strategies.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Ideally, breaking changes should have landed on the <strong>stable branch</strong> of the |
| compiler before they are finalized.</p> |
| <p><a id="guide"></a></p> |
| <h3 id="removing-a-lint"><a class="header" href="#removing-a-lint">Removing a lint</a></h3> |
| <p>Once we have decided to make a "future warning" into a hard error, we need a PR |
| that removes the custom lint. As an example, here are the steps required to |
| remove the <code>overlapping_inherent_impls</code> compatibility lint. First, convert the |
| name of the lint to uppercase (<code>OVERLAPPING_INHERENT_IMPLS</code>) ripgrep through the |
| source for that string. We will basically by converting each place where this |
| lint name is mentioned (in the compiler, we use the upper-case name, and a macro |
| automatically generates the lower-case string; so searching for |
| <code>overlapping_inherent_impls</code> would not find much).</p> |
| <blockquote> |
| <p>NOTE: these exact files don't exist anymore, but the procedure is still the same.</p> |
| </blockquote> |
| <h4 id="remove-the-lint"><a class="header" href="#remove-the-lint">Remove the lint.</a></h4> |
| <p>The first reference you will likely find is the lint definition <a href="https://github.com/rust-lang/rust/blob/085d71c3efe453863739c1fb68fd9bd1beff214f/src/librustc/lint/builtin.rs#L171-L175">in |
| <code>rustc_session/src/lint/builtin.rs</code> that resembles this</a>:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>declare_lint! { |
| pub OVERLAPPING_INHERENT_IMPLS, |
| Deny, // this may also say Warning |
| "two overlapping inherent impls define an item with the same name were erroneously allowed" |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This <code>declare_lint!</code> macro creates the relevant data structures. Remove it. You |
| will also find that there is a mention of <code>OVERLAPPING_INHERENT_IMPLS</code> later in |
| the file as <a href="https://github.com/rust-lang/rust/blob/085d71c3efe453863739c1fb68fd9bd1beff214f/src/librustc/lint/builtin.rs#L252-L290">part of a <code>lint_array!</code></a>; remove it too.</p> |
| <p>Next, you see <a href="https://github.com/rust-lang/rust/blob/085d71c3efe453863739c1fb68fd9bd1beff214f/src/librustc_lint/lib.rs#L202-L205">a reference to <code>OVERLAPPING_INHERENT_IMPLS</code> in |
| <code>rustc_lint/src/lib.rs</code></a>. This is defining the lint as a "future |
| compatibility lint":</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>FutureIncompatibleInfo { |
| id: LintId::of(OVERLAPPING_INHERENT_IMPLS), |
| reference: "issue #36889 <https://github.com/rust-lang/rust/issues/36889>", |
| }, |
| <span class="boring">}</span></code></pre></pre> |
| <p>Remove this too.</p> |
| <h4 id="add-the-lint-to-the-list-of-removed-lints"><a class="header" href="#add-the-lint-to-the-list-of-removed-lints">Add the lint to the list of removed lints.</a></h4> |
| <p>In <code>compiler/rustc_lint/src/lib.rs</code> there is a list of "renamed and removed lints". |
| You can add this lint to the list:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>store.register_removed("overlapping_inherent_impls", "converted into hard error, see #36889"); |
| <span class="boring">}</span></code></pre></pre> |
| <p>where <code>#36889</code> is the tracking issue for your lint.</p> |
| <h4 id="update-the-places-that-issue-the-lint"><a class="header" href="#update-the-places-that-issue-the-lint">Update the places that issue the lint</a></h4> |
| <p>Finally, the last class of references you will see are the places that actually |
| <strong>trigger</strong> the lint itself (i.e., what causes the warnings to appear). These |
| you do not want to delete. Instead, you want to convert them into errors. In |
| this case, the <a href="https://github.com/rust-lang/rust/blob/085d71c3efe453863739c1fb68fd9bd1beff214f/src/librustc_typeck/coherence/inherent.rs#L300-L303"><code>add_lint</code> call</a> looks like this:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>self.tcx.sess.add_lint(lint::builtin::OVERLAPPING_INHERENT_IMPLS, |
| node_id, |
| self.tcx.span_of_impl(item1).unwrap(), |
| msg); |
| <span class="boring">}</span></code></pre></pre> |
| <p>We want to convert this into an error. In some cases, there may be an |
| existing error for this scenario. In others, we will need to allocate a |
| fresh diagnostic code. <a href="./diagnostics/error-codes.html">Instructions for allocating a fresh diagnostic |
| code can be found here.</a> You may want |
| to mention in the extended description that the compiler behavior |
| changed on this point, and include a reference to the tracking issue for |
| the change.</p> |
| <p>Let's say that we've adopted <code>E0592</code> as our code. Then we can change the |
| <code>add_lint()</code> call above to something like:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct_span_code_err!(self.dcx(), self.tcx.span_of_impl(item1).unwrap(), E0592, msg) |
| .emit(); |
| <span class="boring">}</span></code></pre></pre> |
| <h4 id="update-tests"><a class="header" href="#update-tests">Update tests</a></h4> |
| <p>Finally, run the test suite. These should be some tests that used to reference |
| the <code>overlapping_inherent_impls</code> lint, those will need to be updated. In |
| general, if the test used to have <code>#[deny(overlapping_inherent_impls)]</code>, that |
| can just be removed.</p> |
| <pre><code>./x test |
| </code></pre> |
| <h4 id="all-done"><a class="header" href="#all-done">All done!</a></h4> |
| <p>Open a PR. =)</p> |
| <!-- -Links--------------------------------------------------------------------- --> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="using-external-repositories"><a class="header" href="#using-external-repositories">Using External Repositories</a></h1> |
| <p>The <code>rust-lang/rust</code> git repository depends on several other repos in the <code>rust-lang</code> organization. |
| There are three main ways we use dependencies:</p> |
| <ol> |
| <li>As a Cargo dependency through crates.io (e.g. <code>rustc-rayon</code>)</li> |
| <li>As a git (e.g. <code>clippy</code>) or a <a href="https://josh-project.github.io/josh/intro.html">josh</a> (e.g. <code>miri</code>) subtree</li> |
| <li>As a git submodule (e.g. <code>cargo</code>)</li> |
| </ol> |
| <p>As a general rule:</p> |
| <ul> |
| <li>Use crates.io for libraries that could be useful for others in the ecosystem</li> |
| <li>Use subtrees for tools that depend on compiler internals and need to be updated if there are breaking |
| changes</li> |
| <li>Use submodules for tools that are independent of the compiler</li> |
| </ul> |
| <h2 id="external-dependencies-subtrees"><a class="header" href="#external-dependencies-subtrees">External Dependencies (subtrees)</a></h2> |
| <p>The following external projects are managed using some form of a <code>subtree</code>:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust-clippy">clippy</a></li> |
| <li><a href="https://github.com/rust-lang/miri">miri</a></li> |
| <li><a href="https://github.com/rust-lang/portable-simd">portable-simd</a></li> |
| <li><a href="https://github.com/rust-lang/rustfmt">rustfmt</a></li> |
| <li><a href="https://github.com/rust-lang/rust-analyzer">rust-analyzer</a></li> |
| <li><a href="https://github.com/rust-lang/rustc_codegen_cranelift">rustc_codegen_cranelift</a></li> |
| <li><a href="https://github.com/rust-lang/rustc-dev-guide">rustc-dev-guide</a></li> |
| <li><a href="https://github.com/rust-lang/compiler-builtins">compiler-builtins</a></li> |
| <li><a href="https://github.com/rust-lang/stdarch">stdarch</a></li> |
| </ul> |
| <p>In contrast to <code>submodule</code> dependencies |
| (see below for those), the <code>subtree</code> dependencies are just regular files and directories which can |
| be updated in tree. However, if possible, enhancements, bug fixes, etc. specific |
| to these tools should be filed against the tools directly in their respective |
| upstream repositories. The exception is that when rustc changes are required to |
| implement a new tool feature or test, that should happen in one collective rustc PR.</p> |
| <p><code>subtree</code> dependencies are currently managed by two distinct approaches:</p> |
| <ul> |
| <li>Using <code>git subtree</code> |
| <ul> |
| <li><code>clippy</code> (<a href="https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html#performing-the-sync-from-rust-langrust-to-clippy">sync guide</a>)</li> |
| <li><code>portable-simd</code> (<a href="https://github.com/rust-lang/portable-simd/blob/master/subtree-sync.sh">sync script</a>)</li> |
| <li><code>rustfmt</code></li> |
| <li><code>rustc_codegen_cranelift</code> (<a href="https://github.com/rust-lang/rustc_codegen_cranelift/blob/113af154d459e41b3dc2c5d7d878e3d3a8f33c69/scripts/rustup.sh#L7">sync script</a>)</li> |
| </ul> |
| </li> |
| <li>Using the <a href="external-repos.html#synchronizing-a-josh-subtree">josh</a> tool |
| <ul> |
| <li><code>miri</code></li> |
| <li><code>rust-analyzer</code></li> |
| <li><code>rustc-dev-guide</code></li> |
| <li><code>compiler-builtins</code></li> |
| <li><code>stdarch</code></li> |
| </ul> |
| </li> |
| </ul> |
| <h3 id="josh-subtrees"><a class="header" href="#josh-subtrees">Josh subtrees</a></h3> |
| <p>The <a href="https://josh-project.github.io/josh/intro.html">josh</a> tool is an alternative to git subtrees, which manages git history in a different way and scales better to larger repositories. Specific tooling is required to work with josh. We provide a helper <a href="https://github.com/rust-lang/josh-sync"><code>rustc-josh-sync</code></a> tool to help with the synchronization, described <a href="external-repos.html#synchronizing-a-josh-subtree">below</a>.</p> |
| <h3 id="synchronizing-a-josh-subtree"><a class="header" href="#synchronizing-a-josh-subtree">Synchronizing a Josh subtree</a></h3> |
| <p>We use a dedicated tool called <a href="https://github.com/rust-lang/josh-sync"><code>rustc-josh-sync</code></a> for performing Josh subtree updates. |
| The commands below can be used for all our Josh subtrees, although note that <code>miri</code> |
| requires you to perform some <a href="https://github.com/rust-lang/miri/blob/master/CONTRIBUTING.md#advanced-topic-syncing-with-the-rustc-repo">additional steps</a> during pulls.</p> |
| <p>You can install the tool using the following command:</p> |
| <pre><code>cargo install --locked --git https://github.com/rust-lang/josh-sync |
| </code></pre> |
| <p>Both pulls (synchronize changes from rust-lang/rust into the subtree) and pushes (synchronize |
| changes from the subtree to rust-lang/rust) are performed from the subtree repository (so first |
| switch to its repository checkout directory in your terminal).</p> |
| <h4 id="performing-pull"><a class="header" href="#performing-pull">Performing pull</a></h4> |
| <ol> |
| <li>Checkout a new branch that will be used to create a PR into the subtree</li> |
| <li>Run the pull command |
| <pre><code>rustc-josh-sync pull |
| </code></pre> |
| </li> |
| <li>Push the branch to your fork and create a PR into the subtree repository |
| <ul> |
| <li>If you have <code>gh</code> CLI installed, <code>rustc-josh-sync</code> can create the PR for you.</li> |
| </ul> |
| </li> |
| </ol> |
| <h4 id="performing-push"><a class="header" href="#performing-push">Performing push</a></h4> |
| <blockquote> |
| <p>NOTE: |
| Before you proceed, look at some guidance related to Git <a href="https://github.com/rust-lang/josh-sync#git-peculiarities">on josh-sync README</a>.</p> |
| </blockquote> |
| <ol> |
| <li>Run the push command to create a branch named <code><branch-name></code> in a <code>rustc</code> fork under the <code><gh-username></code> account |
| <pre><code>rustc-josh-sync push <branch-name> <gh-username> |
| </code></pre> |
| </li> |
| <li>Create a PR from <code><branch-name></code> into <code>rust-lang/rust</code></li> |
| </ol> |
| <h3 id="creating-a-new-josh-subtree-dependency"><a class="header" href="#creating-a-new-josh-subtree-dependency">Creating a new Josh subtree dependency</a></h3> |
| <p>If you want to migrate a repository dependency from <code>git subtree</code> or <code>git submodule</code> to josh, you can check out <a href="https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg">this guide</a>.</p> |
| <h3 id="synchronizing-a-git-subtree"><a class="header" href="#synchronizing-a-git-subtree">Synchronizing a git subtree</a></h3> |
| <p>Periodically the changes made to subtree based dependencies need to be synchronized between this |
| repository and the upstream tool repositories.</p> |
| <p>Subtree synchronizations are typically handled by the respective tool maintainers. Other users |
| are welcome to submit synchronization PRs, however, in order to do so you will need to modify |
| your local git installation and follow a very precise set of instructions. |
| These instructions are documented, along with several useful tips and tricks, in the |
| <a href="https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html">syncing subtree changes</a> section in Clippy's Contributing guide. |
| The instructions are applicable for use with any subtree based tool, just be sure to |
| use the correct corresponding subtree directory and remote repository.</p> |
| <p>The synchronization process goes in two directions: <code>subtree push</code> and <code>subtree pull</code>.</p> |
| <p>A <code>subtree push</code> takes all the changes that happened to the copy in this repo and creates commits |
| on the remote repo that match the local changes. Every local |
| commit that touched the subtree causes a commit on the remote repo, but |
| is modified to move the files from the specified directory to the tool repo root.</p> |
| <p>A <code>subtree pull</code> takes all changes since the last <code>subtree pull</code> |
| from the tool repo and adds these commits to the rustc repo along with a merge commit that moves |
| the tool changes into the specified directory in the Rust repository.</p> |
| <p>It is recommended that you always do a push first and get that merged to the default branch of the tool. |
| Then, when you do a pull, the merge works without conflicts. |
| While it's definitely possible to resolve conflicts during a pull, you may have to redo the conflict |
| resolution if your PR doesn't get merged fast enough and there are new conflicts. Do not try to |
| rebase the result of a <code>git subtree pull</code>, rebasing merge commits is a bad idea in general.</p> |
| <p>You always need to specify the <code>-P</code> prefix to the subtree directory and the corresponding remote |
| repository. If you specify the wrong directory or repository |
| you'll get very fun merges that try to push the wrong directory to the wrong remote repository. |
| Luckily you can just abort this without any consequences by throwing away either the pulled commits |
| in rustc or the pushed branch on the remote and try again. It is usually fairly obvious |
| that this is happening because you suddenly get thousands of commits that want to be synchronized.</p> |
| <h3 id="creating-a-new-subtree-dependency"><a class="header" href="#creating-a-new-subtree-dependency">Creating a new subtree dependency</a></h3> |
| <p>If you want to create a new subtree dependency from an existing repository, call (from this |
| repository's root directory!)</p> |
| <pre><code>git subtree add -P src/tools/clippy https://github.com/rust-lang/rust-clippy.git master |
| </code></pre> |
| <p>This will create a new commit, which you may not rebase under any circumstances! Delete the commit |
| and redo the operation if you need to rebase.</p> |
| <p>Now you're done, the <code>src/tools/clippy</code> directory behaves as if Clippy were |
| part of the rustc monorepo, so no one but you (or others that synchronize |
| subtrees) actually needs to use <code>git subtree</code>.</p> |
| <h2 id="external-dependencies-submodules"><a class="header" href="#external-dependencies-submodules">External Dependencies (submodules)</a></h2> |
| <p>Building Rust will also use external git repositories tracked using <a href="https://git-scm.com/book/en/v2/Git-Tools-Submodules">git |
| submodules</a>. The complete list may be found in the <a href="https://github.com/rust-lang/rust/blob/HEAD/.gitmodules"><code>.gitmodules</code></a> file. Some |
| of these projects are required (like <code>stdarch</code> for the standard library) and |
| some of them are optional (like <code>src/doc/book</code>).</p> |
| <p>Usage of submodules is discussed more in the <a href="git.html#git-submodules">Using Git chapter</a>.</p> |
| <p>Some of the submodules are allowed to be in a "broken" state where they |
| either don't build or their tests don't pass, e.g. the documentation books |
| like <a href="https://github.com/rust-lang/reference/">The Rust Reference</a>. Maintainers of these projects will be notified |
| when the project is in a broken state, and they should fix them as soon |
| as possible. The current status is tracked on the <a href="https://rust-lang-nursery.github.io/rust-toolstate/">toolstate website</a>. |
| More information may be found on the Forge <a href="https://forge.rust-lang.org/infra/toolstate.html">Toolstate chapter</a>. |
| In practice, it is very rare for documentation to have broken toolstate.</p> |
| <p>Breakage is not allowed in the beta and stable channels, and must be addressed |
| before the PR is merged. They are also not allowed to be broken on <code>main</code> in |
| the week leading up to the beta cut.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="fuzzing"><a class="header" href="#fuzzing">Fuzzing</a></h1> |
| <!-- date-check: Mar 2023 --> |
| <p>For the purposes of this guide, <em>fuzzing</em> is any testing methodology that |
| involves compiling a wide variety of programs in an attempt to uncover bugs in |
| rustc. Fuzzing is often used to find internal compiler errors (ICEs). Fuzzing |
| can be beneficial, because it can find bugs before users run into them and |
| provide small, self-contained programs that make the bug easier to track down. |
| However, some common mistakes can reduce the helpfulness of fuzzing and end up |
| making contributors' lives harder. To maximize your positive impact on the Rust |
| project, please read this guide before reporting fuzzer-generated bugs!</p> |
| <h2 id="guidelines"><a class="header" href="#guidelines">Guidelines</a></h2> |
| <h3 id="in-a-nutshell"><a class="header" href="#in-a-nutshell">In a nutshell</a></h3> |
| <p><em>Please do:</em></p> |
| <ul> |
| <li>Ensure the bug is still present on the latest nightly rustc</li> |
| <li>Include a reasonably minimal, standalone example along with any bug report</li> |
| <li>Include all of the information requested in the bug report template</li> |
| <li>Search for existing reports with the same message and query stack</li> |
| <li>Format the test case with <code>rustfmt</code>, if it maintains the bug</li> |
| <li>Indicate that the bug was found by fuzzing</li> |
| </ul> |
| <p><em>Please don't:</em></p> |
| <ul> |
| <li>Don't report lots of bugs that use internal features, including but not |
| limited to <code>custom_mir</code>, <code>lang_items</code>, <code>no_core</code>, and <code>rustc_attrs</code>.</li> |
| <li>Don't seed your fuzzer with inputs that are known to crash rustc (details |
| below).</li> |
| </ul> |
| <h3 id="discussion-1"><a class="header" href="#discussion-1">Discussion</a></h3> |
| <p>If you're not sure whether or not an ICE is a duplicate of one that's already |
| been reported, please go ahead and report it and link to issues you think might |
| be related. In general, ICEs on the same line but with different <em>query stacks</em> |
| are usually distinct bugs. For example, <a href="https://github.com/rust-lang/rust/issues/109020">#109020</a> and <a href="https://github.com/rust-lang/rust/issues/109129">#109129</a> |
| had similar error messages:</p> |
| <pre><code>error: internal compiler error: compiler/rustc_middle/src/ty/normalize_erasing_regions.rs:195:90: Failed to normalize <[closure@src/main.rs:36:25: 36:28] as std::ops::FnOnce<(Emplacable<()>,)>>::Output, maybe try to call `try_normalize_erasing_regions` instead |
| </code></pre> |
| <pre><code>error: internal compiler error: compiler/rustc_middle/src/ty/normalize_erasing_regions.rs:195:90: Failed to normalize <() as Project>::Assoc, maybe try to call `try_normalize_erasing_regions` instead |
| </code></pre> |
| <p>but different query stacks:</p> |
| <pre><code>query stack during panic: |
| #0 [fn_abi_of_instance] computing call ABI of `<[closure@src/main.rs:36:25: 36:28] as core::ops::function::FnOnce<(Emplacable<()>,)>>::call_once - shim(vtable)` |
| end of query stack |
| </code></pre> |
| <pre><code>query stack during panic: |
| #0 [check_mod_attrs] checking attributes in top-level module |
| #1 [analysis] running analysis passes on this crate |
| end of query stack |
| </code></pre> |
| <h2 id="building-a-corpus"><a class="header" href="#building-a-corpus">Building a corpus</a></h2> |
| <p>When building a corpus, be sure to avoid collecting tests that are already |
| known to crash rustc. A fuzzer that is seeded with such tests is more likely to |
| generate bugs with the same root cause, wasting everyone's time. The simplest |
| way to avoid this is to loop over each file in the corpus, see if it causes an |
| ICE, and remove it if so.</p> |
| <p>To build a corpus, you may want to use:</p> |
| <ul> |
| <li>The rustc/rust-analyzer/clippy test suites (or even source code) --- though avoid |
| tests that are already known to cause failures, which often begin with comments |
| like <code>//@ failure-status: 101</code> or <code>//@ known-bug: #NNN</code>.</li> |
| <li>The already-fixed ICEs in the archived <a href="https://github.com/rust-lang/glacier">Glacier</a> repository --- though |
| avoid the unfixed ones in <code>ices/</code>!</li> |
| </ul> |
| <h2 id="extra-credit"><a class="header" href="#extra-credit">Extra credit</a></h2> |
| <p>Here are a few things you can do to help the Rust project after filing an ICE.</p> |
| <ul> |
| <li><a href="https://rust-lang.github.io/cargo-bisect-rustc/">Bisect</a> the bug to figure out when it was introduced. |
| If you find the regressing PR / commit, you can mark the issue with the label |
| <code>S-has-bisection</code>. If not, consider applying <code>E-needs-bisection</code> instead.</li> |
| <li>Fix "distractions": problems with the test case that don't contribute to |
| triggering the ICE, such as syntax errors or borrow-checking errors</li> |
| <li>Minimize the test case (see below). If successful, you can label the |
| issue with <code>S-has-mcve</code>. Otherwise, you can apply <code>E-needs-mcve</code>.</li> |
| <li>Add the minimal test case to the rust-lang/rust repo as a <a href="tests/compiletest.html#crash-tests">crash test</a>. |
| While you're at it, consider including other "untracked" crashes in your PR. |
| Please don't forget to mark all relevant issues with <code>S-bug-has-test</code> once |
| your PR is merged.</li> |
| </ul> |
| <p>See also <a href="https://forge.rust-lang.org/release/issue-triaging.html#applying-and-removing-labels">applying and removing labels</a>.</p> |
| <h2 id="minimization"><a class="header" href="#minimization">Minimization</a></h2> |
| <p>It is helpful to carefully <em>minimize</em> the fuzzer-generated input. When |
| minimizing, be careful to preserve the original error, and avoid introducing |
| distracting problems such as syntax, type-checking, or borrow-checking errors.</p> |
| <p>There are some tools that can help with minimization. If you're not sure how |
| to avoid introducing syntax, type-, and borrow-checking errors while using |
| these tools, post both the complete and minimized test cases. Generally, |
| <em>syntax-aware</em> tools give the best results in the least amount of time. |
| <a href="https://github.com/langston-barrett/treereduce"><code>treereduce-rust</code></a> and <a href="https://github.com/renatahodovan/picireny">picireny</a> are syntax-aware. |
| <a href="https://github.com/googleprojectzero/halfempty"><code>halfempty</code></a> is not, but is generally a high-quality tool.</p> |
| <h2 id="effective-fuzzing"><a class="header" href="#effective-fuzzing">Effective fuzzing</a></h2> |
| <p>When fuzzing rustc, you may want to avoid generating machine code, since this |
| is mostly done by LLVM. Try <code>--emit=mir</code> instead.</p> |
| <p>A variety of compiler flags can uncover different issues. <code>-Zmir-opt-level=4</code> |
| will turn on MIR optimization passes that are not run by default, potentially |
| uncovering interesting bugs. <code>-Zvalidate-mir</code> can help uncover such bugs.</p> |
| <p>If you're fuzzing a compiler you built, you may want to build it with <code>-C target-cpu=native</code> or even PGO/BOLT to squeeze out a few more executions per |
| second. Of course, it's best to try multiple build configurations and see |
| what actually results in superior throughput.</p> |
| <p>You may want to build rustc from source with debug assertions to find |
| additional bugs, though this is a trade-off: it can slow down fuzzing by |
| requiring extra work for every execution. To enable debug assertions, add this |
| to <code>bootstrap.toml</code> when compiling rustc:</p> |
| <pre><code class="language-toml">[rust] |
| debug-assertions = true |
| </code></pre> |
| <p>ICEs that require debug assertions to reproduce should be tagged |
| <a href="https://github.com/rust-lang/rust/labels/requires-debug-assertions"><code>requires-debug-assertions</code></a>.</p> |
| <h2 id="existing-projects"><a class="header" href="#existing-projects">Existing projects</a></h2> |
| <ul> |
| <li><a href="https://github.com/dwrensha/fuzz-rustc">fuzz-rustc</a> demonstrates how to fuzz rustc with libfuzzer</li> |
| <li><a href="https://github.com/matthiaskrgr/icemaker/">icemaker</a> runs rustc and other tools on a large number of source |
| files with a variety of flags to catch ICEs</li> |
| <li><a href="https://github.com/langston-barrett/tree-splicer/">tree-splicer</a> generates new source files by combining existing |
| ones while maintaining correct syntax</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="notification-groups"><a class="header" href="#notification-groups">Notification groups</a></h1> |
| <p>The <strong>notification groups</strong> are an easy way to help out with rustc in a |
| "piece-meal" fashion, without committing to a larger project. |
| Notification groups are <strong><a href="notification-groups/about.html#join">easy to join</a></strong> (just submit a PR!) |
| and joining does not entail any particular commitment.</p> |
| <p>Once you <a href="notification-groups/about.html#join">join a notification group</a>, you will be added to |
| a list that receives pings on github whenever a new issue is found |
| that fits the notification group's criteria. If you are interested, you |
| can then <a href="https://forge.rust-lang.org/triagebot/issue-assignment.html">claim the issue</a> and start working on it.</p> |
| <p>Of course, you don't have to wait for new issues to be tagged! If you |
| prefer, you can use the Github label for a notification group to |
| search for existing issues that haven't been claimed yet.</p> |
| <h2 id="list-of-notification-groups"><a class="header" href="#list-of-notification-groups">List of notification groups</a></h2> |
| <p>Here's the list of the notification groups:</p> |
| <ul> |
| <li><a href="notification-groups/./apple.html">Apple</a></li> |
| <li><a href="notification-groups/./arm.html">ARM</a></li> |
| <li><a href="notification-groups/./emscripten.html">Emscripten</a></li> |
| <li><a href="notification-groups/./risc-v.html">RISC-V</a></li> |
| <li><a href="notification-groups/./wasi.html">WASI</a></li> |
| <li><a href="notification-groups/./wasm.html">WebAssembly</a></li> |
| <li><a href="notification-groups/./windows.html">Windows</a></li> |
| <li><a href="notification-groups/./rust-for-linux.html">Rust for Linux</a></li> |
| </ul> |
| <h2 id="what-issues-are-a-good-fit-for-notification-groups"><a class="header" href="#what-issues-are-a-good-fit-for-notification-groups">What issues are a good fit for notification groups?</a></h2> |
| <p>Notification groups tend to get pinged on <strong>isolated</strong> bugs, |
| particularly those of <strong>middle priority</strong>:</p> |
| <ul> |
| <li>By <strong>isolated</strong>, we mean that we do not expect large-scale refactoring |
| to be required to fix the bug.</li> |
| <li>By <strong>middle priority</strong>, we mean that we'd like to see the bug fixed, |
| but it's not such a burning problem that we are dropping everything |
| else to fix it. The danger with such bugs, of course, is that they |
| can accumulate over time, and the role of the notification group is |
| to try and stop that from happening!</li> |
| </ul> |
| <p><a id="join"></a></p> |
| <h2 id="joining-a-notification-group"><a class="header" href="#joining-a-notification-group">Joining a notification group</a></h2> |
| <p>To join a notification group, you just have to open a PR adding your |
| Github username to the appropriate file in the Rust team repository. |
| See the "example PRs" below to get a precise idea and to identify the |
| file to edit.</p> |
| <p>Also, if you are not already a member of a Rust team then -- in addition |
| to adding your name to the file -- you have to checkout the repository and |
| run the following command:</p> |
| <pre><code class="language-bash">cargo run add-person $your_user_name |
| </code></pre> |
| <p>Example PRs:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/team/pull/1434">Example of adding yourself to the Apple group.</a></li> |
| <li><a href="https://github.com/rust-lang/team/pull/358">Example of adding yourself to the ARM group.</a></li> |
| <li><a href="https://github.com/rust-lang/team/pull/1579">Example of adding yourself to the Emscripten group.</a></li> |
| <li><a href="https://github.com/rust-lang/team/pull/394">Example of adding yourself to the RISC-V group.</a></li> |
| <li><a href="https://github.com/rust-lang/team/pull/1580">Example of adding yourself to the WASI group.</a></li> |
| <li><a href="https://github.com/rust-lang/team/pull/1581">Example of adding yourself to the WebAssembly group.</a></li> |
| <li><a href="https://github.com/rust-lang/team/pull/348">Example of adding yourself to the Windows group.</a></li> |
| </ul> |
| <h2 id="tagging-an-issue-for-a-notification-group"><a class="header" href="#tagging-an-issue-for-a-notification-group">Tagging an issue for a notification group</a></h2> |
| <p>To tag an issue as appropriate for a notification group, you give |
| <a href="https://github.com/rust-lang/triagebot/">rustbot</a> a <a href="https://forge.rust-lang.org/triagebot/pinging.html"><code>ping</code></a> command with the name of the notification |
| group. For example:</p> |
| <pre><code class="language-text">@rustbot ping apple |
| @rustbot ping arm |
| @rustbot ping emscripten |
| @rustbot ping risc-v |
| @rustbot ping wasi |
| @rustbot ping wasm |
| @rustbot ping windows |
| </code></pre> |
| <p>To make some commands shorter and easier to remember, there are aliases, |
| defined in the <a href="https://github.com/rust-lang/rust/blob/HEAD/triagebot.toml"><code>triagebot.toml</code></a> file. For example, all of these commands |
| are equivalent and will ping the Apple group:</p> |
| <pre><code class="language-text">@rustbot ping apple |
| @rustbot ping macos |
| @rustbot ping ios |
| </code></pre> |
| <p>Keep in mind that these aliases are meant to make humans' life easier. |
| They might be subject to change. If you need to ensure that a command |
| will always be valid, prefer the full invocations over the aliases.</p> |
| <p><strong>Note though that this should only be done by compiler team members |
| or contributors, and is typically done as part of compiler team |
| triage.</strong></p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="apple-notification-group"><a class="header" href="#apple-notification-group">Apple notification group</a></h1> |
| <p><strong>Github Labels:</strong> <a href="https://github.com/rust-lang/rust/labels/O-macos">O-macos</a>, <a href="https://github.com/rust-lang/rust/labels/O-ios">O-ios</a>, <a href="https://github.com/rust-lang/rust/labels/O-tvos">O-tvos</a>, <a href="https://github.com/rust-lang/rust/labels/O-watchos">O-watchos</a> and <a href="https://github.com/rust-lang/rust/labels/O-visionos">O-visionos</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping apple</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| Apple-related issues as well as suggestions on how to resolve interesting |
| questions regarding our macOS/iOS/tvOS/watchOS/visionOS support.</p> |
| <p>To get a better idea for what the group will do, here are some examples of the |
| kinds of questions where we would have reached out to the group for advice in |
| determining the best course of action:</p> |
| <ul> |
| <li>Raising the minimum supported versions (e.g. <a href="https://github.com/rust-lang/rust/pull/104385">#104385</a>)</li> |
| <li>Additional Apple targets (e.g. <a href="https://github.com/rust-lang/rust/pull/121419">#121419</a>)</li> |
| <li>Obscure Xcode linker details (e.g. <a href="https://github.com/rust-lang/rust/pull/121430">#121430</a>)</li> |
| </ul> |
| <h2 id="deployment-targets"><a class="header" href="#deployment-targets">Deployment targets</a></h2> |
| <p>Apple platforms have a concept of "deployment target", controlled with the |
| <code>*_DEPLOYMENT_TARGET</code> environment variables, and specifies the minimum OS |
| version that a binary runs on.</p> |
| <p>Using an API from a newer OS version in the standard library than the default |
| that <code>rustc</code> uses will result in either a static or a dynamic linker error. |
| For this reason, try to suggest that people document on <code>extern "C"</code> APIs |
| which OS version they were introduced with, and if that's newer than the |
| current default used by <code>rustc</code>, suggest to use weak linking.</p> |
| <h2 id="the-app-store-and-private-apis"><a class="header" href="#the-app-store-and-private-apis">The App Store and private APIs</a></h2> |
| <p>Apple are very protective about using undocumented APIs, so it's important |
| that whenever a change uses a new function, that they are verified to actually |
| be public API, as even just mentioning undocumented APIs in the binary |
| (without calling it) can lead to rejections from the App Store.</p> |
| <p>For example, Darwin / the XNU kernel actually has futex syscalls, but we can't |
| use them in <code>std</code> because they are not public API.</p> |
| <p>In general, for an API to be considered public by Apple, it has to:</p> |
| <ul> |
| <li>Appear in a public header (i.e. one distributed with Xcode, and found for |
| the specific platform under <code>xcrun --show-sdk-path --sdk $SDK</code>).</li> |
| <li>Have an availability attribute on it (like <code>__API_AVAILABLE</code>, |
| <code>API_AVAILABLE</code> or similar).</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="arm-notification-group"><a class="header" href="#arm-notification-group">ARM notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-ARM">O-ARM</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping arm</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| ARM-related issues as well as suggestions on how to resolve |
| interesting questions regarding our ARM support.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#narrow/stream/242906-t-compiler.2Farm"><code>#t-compiler/arm</code></a>) |
| where people can go to pose questions and discuss ARM-specific |
| topics.</p> |
| <p>So, if you are interested in participating, please sign up for the |
| ARM group! To do so, open a PR against the <a href="https://github.com/rust-lang/team">rust-lang/team</a> |
| repository. Just <a href="https://github.com/rust-lang/team/pull/358">follow this example</a>, but change the username to |
| your own!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="emscripten-notification-group"><a class="header" href="#emscripten-notification-group">Emscripten notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-emscripten">O-emscripten</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping emscripten</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| Emscripten-related issues as well as suggestions on how to resolve |
| interesting questions regarding our Emscripten support.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#narrow/stream/463513-t-compiler.2Fwasm"><code>#t-compiler/wasm</code></a>) |
| where people can go to pose questions and discuss Emscripten-specific |
| topics.</p> |
| <p>So, if you are interested in participating, please sign up for the |
| Emscripten group! To do so, open a PR against the <a href="https://github.com/rust-lang/team">rust-lang/team</a> |
| repository. Just <a href="https://github.com/rust-lang/team/pull/1579">follow this example</a>, but change the username to |
| your own!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="fuchsia-notification-group"><a class="header" href="#fuchsia-notification-group">Fuchsia notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-fuchsia">O-fuchsia</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping fuchsia</code></p> |
| <p>This list will be used to notify <a href="notification-groups/../tests/ecosystem-test-jobs/fuchsia.html">Fuchsia</a> maintainers |
| when the compiler or the standard library changes in a way that would |
| break the Fuchsia integration.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="risc-v-notification-group"><a class="header" href="#risc-v-notification-group">RISC-V notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-riscv">O-riscv</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping risc-v</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| RISC-V-related issues as well as suggestions on how to resolve |
| interesting questions regarding our RISC-V support.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#narrow/stream/250483-t-compiler.2Frisc-v"><code>#t-compiler/risc-v</code></a>) |
| where people can go to pose questions and discuss RISC-V-specific |
| topics.</p> |
| <p>So, if you are interested in participating, please sign up for the |
| RISC-V group! To do so, open a PR against the <a href="https://github.com/rust-lang/team">rust-lang/team</a> |
| repository. Just <a href="https://github.com/rust-lang/team/pull/394">follow this example</a>, but change the username to |
| your own!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rust-for-linux-notification-group"><a class="header" href="#rust-for-linux-notification-group">Rust for Linux notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/A-rust-for-linux">A-rust-for-linux</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping rfl</code></p> |
| <p>This list will be used to notify <a href="https://rust-for-linux.com/">Rust for Linux (RfL)</a> maintainers |
| when the compiler or the standard library changes in a way that would |
| break Rust for Linux, since it depends on several unstable flags |
| and features. The RfL maintainers should then ideally provide support |
| for resolving the breakage or decide to temporarily accept the breakage |
| and unblock CI by temporarily removing the RfL CI jobs.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#narrow/stream/425075-rust-for-linux"><code>#rust-for-linux</code></a>) |
| where people can go to ask questions and discuss topics related to Rust |
| for Linux.</p> |
| <p>If you are interested in participating, please sign up for the |
| Rust for Linux group on <a href="https://rust-lang.zulipchat.com/#narrow/stream/425075-rust-for-linux">Zulip</a>!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="wasi-notification-group"><a class="header" href="#wasi-notification-group">WASI notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-wasi">O-wasi</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping wasi</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| WASI-related issues as well as suggestions on how to resolve |
| interesting questions regarding our WASI support.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#narrow/stream/463513-t-compiler.2Fwasm"><code>#t-compiler/wasm</code></a>) |
| where people can go to pose questions and discuss WASI-specific |
| topics.</p> |
| <p>So, if you are interested in participating, please sign up for the |
| WASI group! To do so, open a PR against the <a href="https://github.com/rust-lang/team">rust-lang/team</a> |
| repository. Just <a href="https://github.com/rust-lang/team/pull/1580">follow this example</a>, but change the username to |
| your own!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="webassembly-wasm-notification-group"><a class="header" href="#webassembly-wasm-notification-group">WebAssembly (WASM) notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-wasm">O-wasm</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping wasm</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| WebAssembly-related issues as well as suggestions on how to resolve |
| interesting questions regarding our WASM support.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#narrow/stream/463513-t-compiler.2Fwasm"><code>#t-compiler/wasm</code></a>) |
| where people can go to pose questions and discuss WASM-specific |
| topics.</p> |
| <p>So, if you are interested in participating, please sign up for the |
| WASM group! To do so, open a PR against the <a href="https://github.com/rust-lang/team">rust-lang/team</a> |
| repository. Just <a href="https://github.com/rust-lang/team/pull/1581">follow this example</a>, but change the username to |
| your own!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="windows-notification-group"><a class="header" href="#windows-notification-group">Windows notification group</a></h1> |
| <p><strong>Github Label:</strong> <a href="https://github.com/rust-lang/rust/labels/O-Windows">O-Windows</a> <br> |
| <strong>Ping command:</strong> <code>@rustbot ping windows</code></p> |
| <p>This list will be used to ask for help both in diagnosing and testing |
| Windows-related issues as well as suggestions on how to resolve |
| interesting questions regarding our Windows support.</p> |
| <p>The group also has an associated Zulip channel (<a href="https://rust-lang.zulipchat.com/#streams/242869/t-compiler.2Fwindows"><code>#t-compiler/windows</code></a>) |
| where people can go to pose questions and discuss Windows-specific |
| topics.</p> |
| <p>To get a better idea for what the group will do, here are some |
| examples of the kinds of questions where we would have reached out to |
| the group for advice in determining the best course of action:</p> |
| <ul> |
| <li>Which versions of MinGW should we support?</li> |
| <li>Should we remove the legacy InnoSetup GUI installer? <a href="https://github.com/rust-lang/rust/pull/72569">#72569</a></li> |
| <li>What names should we use for static libraries on Windows? <a href="https://github.com/rust-lang/rust/pull/29520">#29520</a></li> |
| </ul> |
| <p>So, if you are interested in participating, please sign up for the |
| Windows group! To do so, open a PR against the <a href="https://github.com/rust-lang/team">rust-lang/team</a> |
| repository. Just <a href="https://github.com/rust-lang/team/pull/348/">follow this example</a>, but change the username to |
| your own!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rust-langrust-licenses"><a class="header" href="#rust-langrust-licenses"><code>rust-lang/rust</code> Licenses</a></h1> |
| <p>The <code>rustc</code> compiler source and standard library are dual licensed under the <a href="https://github.com/rust-lang/rust/blob/HEAD/LICENSE-APACHE">Apache License v2.0</a> and the <a href="https://github.com/rust-lang/rust/blob/HEAD/LICENSE-MIT">MIT License</a> unless otherwise specified.</p> |
| <p>Detailed licensing information is available in the <a href="https://github.com/rust-lang/rust/blob/HEAD/COPYRIGHT">COPYRIGHT document</a> of the <code>rust-lang/rust</code> repository.</p> |
| <h2 id="guidelines-for-reviewers"><a class="header" href="#guidelines-for-reviewers">Guidelines for reviewers</a></h2> |
| <p>In general, reviewers need to be looking not only for the code quality of contributions but also |
| that they are properly licensed. |
| We have some tips below for things to look out for when reviewing, but if you ever feel uncertain |
| as to whether some code might be properly licensed, err on the safe side — reach out to the Council |
| or Compiler Team Leads for feedback!</p> |
| <p>Things to watch out for:</p> |
| <ul> |
| <li>The PR author states that they copied, ported, or adapted the code from some other source.</li> |
| <li>There is a comment in the code pointing to a webpage or describing where the algorithm was taken |
| from.</li> |
| <li>The algorithm or code pattern seems like it was likely copied from somewhere else.</li> |
| <li>When adding new dependencies, double check the dependency's license.</li> |
| </ul> |
| <p>In all of these cases, we will want to check that source to make sure it is licensed in a way |
| that is compatible with Rust’s license.</p> |
| <p>Examples</p> |
| <ul> |
| <li>Porting C code from a GPL project, like GNU binutils, is not allowed. That would require Rust |
| itself to be licensed under the GPL.</li> |
| <li>Copying code from an algorithms text book may be allowed, but some algorithms are patented.</li> |
| </ul> |
| <h2 id="porting"><a class="header" href="#porting">Porting</a></h2> |
| <p>Contributions to rustc, especially around platform and compiler intrinsics, often include porting |
| over work from other projects, mainly LLVM and GCC.</p> |
| <p>Some general rules apply:</p> |
| <ul> |
| <li>Copying work needs to adhere to the original license |
| <ul> |
| <li>This applies to direct copy & paste</li> |
| <li>This also applies to code you looked at and ported</li> |
| </ul> |
| </li> |
| </ul> |
| <p>In general, taking inspiration from other codebases is fine, but please exercise caution when |
| porting code.</p> |
| <p>Ports of full libraries (e.g. C libraries shipped with LLVM) must keep the license of the original |
| library.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="editions"><a class="header" href="#editions">Editions</a></h1> |
| <p>This chapter gives an overview of how Edition support works in rustc. |
| This assumes that you are familiar with what Editions are (see the <a href="https://doc.rust-lang.org/edition-guide/">Edition Guide</a>).</p> |
| <h2 id="edition-definition"><a class="header" href="#edition-definition">Edition definition</a></h2> |
| <p>The <code>--edition</code> CLI flag specifies the edition to use for a crate. |
| This can be accessed from <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html#method.edition"><code>Session::edition</code></a>. |
| There are convenience functions like <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html#method.at_least_rust_2021"><code>Session::at_least_rust_2021</code></a> for checking the crate's |
| edition, though you should be careful about whether you check the global session or the span, see |
| <a href="guides/editions.html#edition-hygiene">Edition hygiene</a> below.</p> |
| <p>As an alternative to the <code>at_least_rust_20xx</code> convenience methods, the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/enum.Edition.html"><code>Edition</code></a> type also |
| supports comparisons for doing range checks, such as <code>span.edition() >= Edition::Edition2021</code>.</p> |
| <h3 id="adding-a-new-edition"><a class="header" href="#adding-a-new-edition">Adding a new edition</a></h3> |
| <p>Adding a new edition mainly involves adding a variant to the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/enum.Edition.html"><code>Edition</code></a> enum and then fixing |
| everything that is broken. See <a href="https://github.com/rust-lang/rust/pull/94461">#94461</a> for an |
| example.</p> |
| <h3 id="features-and-edition-stability"><a class="header" href="#features-and-edition-stability">Features and Edition stability</a></h3> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/enum.Edition.html"><code>Edition</code></a> enum defines whether or not an edition is stable. |
| If it is not stable, then the <code>-Zunstable-options</code> CLI option must be passed to enable it.</p> |
| <p>When adding a new feature, there are two options you can choose for how to handle stability with a |
| future edition:</p> |
| <ul> |
| <li>Just check the edition of the span like <code>span.at_least_rust_20xx()</code> (see <a href="guides/editions.html#edition-hygiene">Edition hygiene</a>) or the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html#method.edition"><code>Session::edition</code></a>. This will implicitly depend on the stability of the edition itself to |
| indicate that your feature is available.</li> |
| <li>Place your new behavior behind a <a href="guides/../feature-gates.html">feature gate</a>.</li> |
| </ul> |
| <p>It may be sufficient to only check the current edition for relatively simple changes. |
| However, for larger language changes, you should consider creating a feature gate. |
| There are several benefits to using a feature gate:</p> |
| <ul> |
| <li>A feature gate makes it easier to work on and experiment with a new feature.</li> |
| <li>It makes the intent clear when the <code>#![feature(…)]</code> attribute is used that your new feature is |
| being enabled.</li> |
| <li>It makes testing of editions easier so that features that are not yet complete do not interfere |
| with testing of edition-specific features that are complete and ready.</li> |
| <li>It decouples the feature from an edition, which makes it easier for the team to make a deliberate |
| decision of whether or not a feature should be added to the next edition when the feature is |
| ready.</li> |
| </ul> |
| <p>When a feature is complete and ready, the feature gate can be removed (and the code should just |
| check the span or <code>Session</code> edition to determine if it is enabled).</p> |
| <p>There are a few different options for doing feature checks:</p> |
| <ul> |
| <li> |
| <p>For highly experimental features, that may or may not be involved in an edition, they can |
| implement regular feature gates like <code>tcx.features().my_feature</code>, and ignore editions for the time |
| being.</p> |
| </li> |
| <li> |
| <p>For experimental features that <em>might</em> be involved in an edition, they should implement gates with |
| <code>tcx.features().my_feature && span.at_least_rust_20xx()</code>. |
| This requires the user to still specify <code>#![feature(my_feature)]</code>, to avoid disrupting testing of |
| other edition features which are ready and have been accepted within the edition.</p> |
| </li> |
| <li> |
| <p>For experimental features that have graduated to definitely be part of an edition, |
| they should implement gates with <code>tcx.features().my_feature || span.at_least_rust_20xx()</code>, |
| or just remove the feature check altogether and just check <code>span.at_least_rust_20xx()</code>.</p> |
| </li> |
| </ul> |
| <p>If you need to do the feature gating in multiple places, consider placing the check in a single |
| function so that there will only be a single place to update. For example:</p> |
| <pre><code class="language-rust ignore">// An example from Edition 2021 disjoint closure captures. |
| |
| fn enable_precise_capture(tcx: TyCtxt<'_>, span: Span) -> bool { |
| tcx.features().capture_disjoint_fields || span.rust_2021() |
| }</code></pre> |
| <p>See <a href="guides/editions.html#lints-and-stability">Lints and stability</a> below for more information about how lints handle |
| stability.</p> |
| <h2 id="edition-parsing"><a class="header" href="#edition-parsing">Edition parsing</a></h2> |
| <p>For the most part, the lexer is edition-agnostic. |
| Within <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html"><code>Lexer</code></a>, tokens can be modified based on edition-specific behavior. |
| For example, C-String literals like <code>c"foo"</code> are split into multiple tokens in editions before 2021. |
| This is also where things like reserved prefixes are handled for the 2021 edition.</p> |
| <p>Edition-specific parsing is relatively rare. One example is <code>async fn</code> which checks the span of the |
| token to determine if it is the 2015 edition, and emits an error in that case. |
| This can only be done if the syntax was already invalid.</p> |
| <p>If you need to do edition checking in the parser, you will normally want to look at the edition of |
| the token, see <a href="guides/editions.html#edition-hygiene">Edition hygiene</a>. |
| In some rare cases you may instead need to check the global edition from <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html#structfield.edition"><code>ParseSess::edition</code></a>.</p> |
| <p>Most edition-specific parsing behavior is handled with <a href="guides/editions.html#migration-lints">migration lints</a> instead of in the parser. |
| This is appropriate when there is a <em>change</em> in syntax (as opposed to new syntax). |
| This allows the old syntax to continue to work on previous editions. |
| The lint then checks for the change in behavior. |
| On older editions, the lint pass should emit the migration lint to help with migrating to new |
| editions. |
| On newer editions, your code should emit a hard error with <code>emit_err</code> instead. |
| For example, the deprecated <code>start...end</code> pattern syntax emits the |
| <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/warn-by-default.html#ellipsis-inclusive-range-patterns"><code>ellipsis_inclusive_range_patterns</code></a> lint on editions before 2021, and in 2021 is an hard error via |
| the <code>emit_err</code> method.</p> |
| <h3 id="keywords"><a class="header" href="#keywords">Keywords</a></h3> |
| <p>New keywords can be introduced across an edition boundary. |
| This is implemented by functions like <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html#method.is_used_keyword_conditional"><code>Symbol::is_used_keyword_conditional</code></a>, which rely on the |
| ordering of how the keywords are defined.</p> |
| <p>When new keywords are introduced, the <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/allowed-by-default.html#keyword-idents"><code>keyword_idents</code></a> lint should be updated so that automatic |
| migrations can transition code that might be using the keyword as an identifier (see |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/builtin/struct.KeywordIdents.html"><code>KeywordIdents</code></a>). |
| An alternative to consider is to implement the keyword as a weak keyword if the position it is used |
| is sufficient to distinguish it.</p> |
| <p>An additional option to consider is the <code>k#</code> prefix which was introduced in <a href="https://rust-lang.github.io/rfcs/3101-reserved_prefixes.html">RFC 3101</a>. |
| This allows the use of a keyword in editions <em>before</em> the edition where the keyword is introduced. |
| This is currently not implemented.</p> |
| <h3 id="edition-hygiene-1"><a class="header" href="#edition-hygiene-1">Edition hygiene</a></h3> |
| <p>Spans are marked with the edition of the crate that the span came from. |
| See <a href="https://doc.rust-lang.org/nightly/edition-guide/editions/advanced-migrations.html#macro-hygiene">Macro hygiene</a> in the Edition Guide for a user-centric description of what this means.</p> |
| <p>You should normally use the edition from the token span instead of looking at the global <code>Session</code> |
| edition. |
| For example, use <code>span.edition().at_least_rust_2021()</code> instead of <code>sess.at_least_rust_2021()</code>. |
| This helps ensure that macros behave correctly when used across crates.</p> |
| <h2 id="lints"><a class="header" href="#lints">Lints</a></h2> |
| <p>Lints support a few different options for interacting with editions. |
| Lints can be <em>future incompatible edition migration lints</em>, which are used to support |
| <a href="guides/editions.html#migration-lints">migrations</a> to newer editions. |
| Alternatively, lints can be <a href="guides/editions.html#edition-specific-lints">edition-specific</a>, where they change their |
| default level starting in a specific edition.</p> |
| <h3 id="migration-lints"><a class="header" href="#migration-lints">Migration lints</a></h3> |
| <p><em>Migration lints</em> are used to migrate projects from one edition to the next. |
| They are implemented with a <code>MachineApplicable</code> <a href="guides/../diagnostics.html#suggestions">suggestion</a> which |
| will rewrite code so that it will <strong>successfully compile in both the previous and the next |
| edition</strong>. |
| For example, the <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/allowed-by-default.html#keyword-idents"><code>keyword_idents</code></a> lint will take identifiers that conflict with a new keyword to |
| use the raw identifier syntax to avoid the conflict (for example changing <code>async</code> to <code>r#async</code>).</p> |
| <p>Migration lints must be declared with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/enum.FutureIncompatibilityReason.html#variant.EditionError"><code>FutureIncompatibilityReason::EditionError</code></a> or |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/enum.FutureIncompatibilityReason.html#variant.EditionSemanticsChange"><code>FutureIncompatibilityReason::EditionSemanticsChange</code></a> <a href="guides/../diagnostics.html#future-incompatible-lints">future-incompatible |
| option</a> in the lint declaration:</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| pub KEYWORD_IDENTS, |
| Allow, |
| "detects edition keywords being used as an identifier", |
| @future_incompatible = FutureIncompatibleInfo { |
| reason: FutureIncompatibilityReason::EditionError(Edition::Edition2018), |
| reference: "issue #49716 <https://github.com/rust-lang/rust/issues/49716>", |
| }; |
| }</code></pre> |
| <p>When declared like this, the lint is automatically added to the appropriate |
| <code>rust-20xx-compatibility</code> lint group. |
| When a user runs <code>cargo fix --edition</code>, cargo will pass the <code>--force-warn rust-20xx-compatibility</code> |
| flag to force all of these lints to appear during the edition migration. |
| Cargo also passes <code>--cap-lints=allow</code> so that no other lints interfere with the edition migration.</p> |
| <p>Make sure that the example code sets the correct edition. The example should illustrate the previous edition, and show what the migration warning would look like. For example, this lint for a 2024 migration shows an example in 2021:</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| /// The `keyword_idents_2024` lint detects ... |
| /// |
| /// ### Example |
| /// |
| /// ```rust,edition2021 |
| /// #![warn(keyword_idents_2024)] |
| /// fn gen() {} |
| /// ``` |
| /// |
| /// {{produces}} |
| }</code></pre> |
| <p>Migration lints can be either <code>Allow</code> or <code>Warn</code> by default. |
| If it is <code>Allow</code>, users usually won't see this warning unless they are doing an edition migration |
| manually or there is a problem during the migration. |
| Most migration lints are <code>Allow</code>.</p> |
| <p>If it is <code>Warn</code> by default, users on all editions will see this warning. |
| Only use <code>Warn</code> if you think it is important for everyone to be aware of the change, and to |
| encourage people to update their code on all editions. |
| Beware that new warn-by-default lint that hit many projects can be very disruptive and frustrating |
| for users. |
| You may consider switching an <code>Allow</code> to <code>Warn</code> several years after the edition stabilizes. |
| This will only show up for the relatively small number of stragglers who have not updated to the new |
| edition.</p> |
| <h3 id="edition-specific-lints"><a class="header" href="#edition-specific-lints">Edition-specific lints</a></h3> |
| <p>Lints can be marked so that they have a different level starting in a specific edition. |
| In the lint declaration, use the <code>@edition</code> marker:</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| pub SOME_LINT_NAME, |
| Allow, |
| "my lint description", |
| @edition Edition2024 => Warn; |
| }</code></pre> |
| <p>Here, <code>SOME_LINT_NAME</code> defaults to <code>Allow</code> on all editions before 2024, and then becomes <code>Warn</code> |
| afterwards.</p> |
| <p>This should generally be used sparingly, as there are other options:</p> |
| <ul> |
| <li> |
| <p>Small impact stylistic changes unrelated to an edition can just make the lint <code>Warn</code> on all |
| editions. If you want people to adopt a different way to write things, then go ahead and commit to |
| having it show up for all projects.</p> |
| <p>Beware that if a new warn-by-default lint hits many projects, it can be very disruptive and |
| frustrating for users.</p> |
| </li> |
| <li> |
| <p>Change the new style to be a hard error in the new edition, and use a <a href="guides/editions.html#migration-lints">migration lint</a> to |
| automatically convert projects to the new style. For example, |
| <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/warn-by-default.html#ellipsis-inclusive-range-patterns"><code>ellipsis_inclusive_range_patterns</code></a> is a hard error in 2021, and warns in all previous editions.</p> |
| <p>Beware that these cannot be added after the edition stabilizes.</p> |
| </li> |
| <li> |
| <p>Migration lints can also change over time. |
| For example, the migration lint can start out as <code>Allow</code> by default. |
| For people performing the migration, they will automatically get updated to the new code. |
| Then, after some years, the lint can be made to <code>Warn</code> in previous editions.</p> |
| <p>For example <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/warn-by-default.html#anonymous-parameters"><code>anonymous_parameters</code></a> was a 2018 Edition migration lint (and a hard-error in 2018) |
| that was <code>Allow</code> by default in previous editions. |
| Then, three years later, it was changed to <code>Warn</code> for all previous editions, so that all users got |
| a warning that the style was being phased out. |
| If this was a warning from the start, it would have impacted many projects and be very disruptive. |
| By making it part of the edition, most users eventually updated to the new edition and were |
| handled by the migration. |
| Switching to <code>Warn</code> only impacted a few stragglers who did not update.</p> |
| </li> |
| </ul> |
| <h3 id="lints-and-stability"><a class="header" href="#lints-and-stability">Lints and stability</a></h3> |
| <p>Lints can be marked as being unstable, which can be helpful when developing a new edition feature, |
| and you want to test out a migration lint. |
| The feature gate can be specified in the lint's declaration like this:</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| pub SOME_LINT_NAME, |
| Allow, |
| "my cool lint", |
| @feature_gate = sym::my_feature_name; |
| }</code></pre> |
| <p>Then, the lint will only fire if the user has the appropriate <code>#![feature(my_feature_name)]</code>. |
| Just beware that when it comes time to do crater runs testing the migration that the feature gate |
| will need to be removed.</p> |
| <p>Alternatively, you can implement an allow-by-default <a href="guides/editions.html#migration-lints">migration lint</a> for an upcoming unstable |
| edition without a feature gate. |
| Although users may technically be able to enable the lint before the edition is stabilized, most |
| will not notice the new lint exists, and it should not disrupt anything or cause any breakage.</p> |
| <h3 id="idiom-lints"><a class="header" href="#idiom-lints">Idiom lints</a></h3> |
| <p>In the 2018 edition, there was a concept of "idiom lints" under the <code>rust-2018-idioms</code> lint group. |
| The concept was to have new idiomatic styles under a different lint group separate from the forced |
| migrations under the <code>rust-2018-compatibility</code> lint group, giving some flexibility as to how people |
| opt-in to certain edition changes.</p> |
| <p>Overall this approach did not seem to work very well, |
| and it is unlikely that we will use the idiom groups in the future.</p> |
| <h2 id="standard-library-changes"><a class="header" href="#standard-library-changes">Standard library changes</a></h2> |
| <h3 id="preludes"><a class="header" href="#preludes">Preludes</a></h3> |
| <p>Each edition comes with a specific prelude of the standard library. |
| These are implemented as regular modules in <a href="https://doc.rust-lang.org/core/prelude/index.html"><code>core::prelude</code></a> and <a href="https://doc.rust-lang.org/std/prelude/index.html"><code>std::prelude</code></a>. |
| New items can be added to the prelude, just beware that this can conflict with user's pre-existing |
| code. |
| Usually a <a href="guides/editions.html#migration-lints">migration lint</a> should be used to migrate existing code to avoid the conflict. |
| For example, <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/allowed-by-default.html#rust-2021-prelude-collisions"><code>rust_2021_prelude_collisions</code></a> is used to handle the collisions with the new traits |
| in 2021.</p> |
| <h3 id="customized-language-behavior"><a class="header" href="#customized-language-behavior">Customized language behavior</a></h3> |
| <p>Usually it is not possible to make breaking changes to the standard library. |
| In some rare cases, the teams may decide that the behavior change is important enough to break this |
| rule. |
| The downside is that this requires special handling in the compiler to be able to distinguish when |
| the old and new signatures or behaviors should be used.</p> |
| <p>One example is the change in method resolution for <a href="https://doc.rust-lang.org/nightly/edition-guide/rust-2021/IntoIterator-for-arrays.html"><code>into_iter()</code> of arrays</a>. |
| This was implemented with the <code>#[rustc_skip_array_during_method_dispatch]</code> attribute on the |
| <code>IntoIterator</code> trait which then tells the compiler to consider an alternate trait resolution choice |
| based on the edition.</p> |
| <p>Another example is the <a href="https://doc.rust-lang.org/nightly/edition-guide/rust-2021/panic-macro-consistency.html"><code>panic!</code> macro changes</a>. |
| This required defining multiple panic macros, and having the built-in panic macro implementation |
| determine the appropriate way to expand it. |
| This also included the <a href="https://doc.rust-lang.org/nightly/rustc/lints/listing/warn-by-default.html#non-fmt-panics"><code>non_fmt_panics</code></a> <a href="guides/editions.html#migration-lints">migration lint</a> to adjust old code to the new form, which |
| required the <code>rustc_diagnostic_item</code> attribute to detect the usage of the panic macro.</p> |
| <p>In general it is recommended to avoid these special cases except for very high value situations.</p> |
| <h3 id="migrating-the-standard-library-edition"><a class="header" href="#migrating-the-standard-library-edition">Migrating the standard library edition</a></h3> |
| <p>Updating the edition of the standard library itself roughly involves the following process:</p> |
| <ul> |
| <li>Wait until the newly stabilized edition has reached beta and the bootstrap compiler has been updated.</li> |
| <li>Apply migration lints. This can be an involved process since some code is in external submodules<sup class="footnote-reference" id="fr-std-submodules-1"><a href="#footnote-std-submodules">1</a></sup>, and the standard library makes heavy use of conditional compilation. Also, running <code>cargo fix --edition</code> can be impractical on the standard library itself. One approach is to individually add <code>#![warn(...)]</code> at the top of each crate for each lint, run <code>./x check library</code>, apply the migrations, remove the <code>#![warn(...)]</code> and commit each migration separately. You'll likely need to run <code>./x check</code> with <code>--target</code> for many different targets to get full coverage (otherwise you'll likely spend days or weeks getting CI to pass)<sup class="footnote-reference" id="fr-ed-docker-1"><a href="#footnote-ed-docker">2</a></sup>. See also the <a href="https://doc.rust-lang.org/nightly/edition-guide/editions/advanced-migrations.html">advanced migration guide</a> for more tips. |
| <ul> |
| <li>Apply migrations to <a href="https://github.com/rust-lang/backtrace-rs/"><code>backtrace-rs</code></a>. <a href="https://github.com/rust-lang/backtrace-rs/pull/700">Example for 2024</a>. Note that this doesn't update the edition of the crate itself because that is published independently on crates.io, and that would otherwise restrict the minimum Rust version. Consider adding some <code>#![deny()]</code> attributes to avoid regressions until its edition gets updated.</li> |
| <li>Apply migrations to <a href="https://github.com/rust-lang/stdarch/"><code>stdarch</code></a>, and update its edition, and formatting. <a href="https://github.com/rust-lang/stdarch/pull/1710">Example for 2024</a>.</li> |
| <li>Post PRs to update the backtrace and stdarch submodules, and wait for those to land.</li> |
| <li>Apply migration lints to the standard library crates, and update their edition. I recommend working one crate at a time starting with <code>core</code>. <a href="https://github.com/rust-lang/rust/pull/138162">Example for 2024</a>.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="stabilizing-an-edition"><a class="header" href="#stabilizing-an-edition">Stabilizing an edition</a></h2> |
| <p>After the edition team has given the go-ahead, the process for stabilizing an edition is roughly:</p> |
| <ul> |
| <li>Update <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/constant.LATEST_STABLE_EDITION.html"><code>LATEST_STABLE_EDITION</code></a>.</li> |
| <li>Update <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/enum.Edition.html#method.is_stable"><code>Edition::is_stable</code></a>.</li> |
| <li>Hunt and find any document that refers to edition by number, and update it: |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/src/doc/rustc/src/command-line-arguments.md#--edition-specify-the-edition-to-use"><code>--edition</code> flag</a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/src/doc/rustdoc/src/write-documentation/documentation-tests.md#attributes">Rustdoc attributes</a></li> |
| </ul> |
| </li> |
| <li>Clean up any tests that use the <code>//@ edition</code> header to remove the <code>-Zunstable-options</code> flag to ensure they are indeed stable. Note: Ideally this should be automated, see <a href="https://github.com/rust-lang/rust/issues/133582">#133582</a>.</li> |
| <li>Bless any tests that change.</li> |
| <li>Update <code>lint-docs</code> to default to the new edition.</li> |
| </ul> |
| <p>See <a href="https://github.com/rust-lang/rust/pull/133349">example for 2024</a>.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-std-submodules"> |
| <p>This will hopefully change in the future to pull these submodules into <code>rust-lang/rust</code>. <a href="#fr-std-submodules-1">↩</a></p> |
| </li> |
| <li id="footnote-ed-docker"> |
| <p>You'll also likely need to do a lot of testing for different targets, and this is where <a href="guides/../tests/docker.html">docker testing</a> comes in handy. <a href="#fr-ed-docker-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="bootstrapping-the-compiler"><a class="header" href="#bootstrapping-the-compiler">Bootstrapping the compiler</a></h1> |
| <p><a href="https://en.wikipedia.org/wiki/Bootstrapping_(compilers)"><em>Bootstrapping</em></a> is the process of using a compiler to compile itself. |
| More accurately, it means using an older compiler to compile a newer version |
| of the same compiler.</p> |
| <p>This raises a chicken-and-egg paradox: where did the first compiler come from? |
| It must have been written in a different language. In Rust's case it was |
| <a href="https://github.com/rust-lang/rust/tree/ef75860a0a72f79f97216f8aaa5b388d98da6480/src/boot">written in OCaml</a>. However, it was abandoned long ago, and the |
| only way to build a modern version of rustc is with a slightly less modern |
| version.</p> |
| <p>This is exactly how <code>x.py</code> works: it downloads the current beta release of |
| rustc, then uses it to compile the new compiler.</p> |
| <p>In this section, we give a high-level overview of |
| <a href="building/bootstrapping/./what-bootstrapping-does.html">what Bootstrap does</a>, followed by a high-level |
| introduction to <a href="building/bootstrapping/./how-bootstrap-does-it.html">how Bootstrap does it</a>.</p> |
| <p>Additionally, see <a href="building/bootstrapping/./debugging-bootstrap.html">debugging bootstrap</a> to learn |
| about debugging methods.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="what-bootstrapping-does"><a class="header" href="#what-bootstrapping-does">What Bootstrapping does</a></h1> |
| <p><a href="https://en.wikipedia.org/wiki/Bootstrapping_(compilers)"><em>Bootstrapping</em></a> is the process of using a compiler to compile itself. |
| More accurately, it means using an older compiler to compile a newer version of |
| the same compiler.</p> |
| <p>This raises a chicken-and-egg paradox: where did the first compiler come from? |
| It must have been written in a different language. In Rust's case it was |
| <a href="https://github.com/rust-lang/rust/tree/ef75860a0a72f79f97216f8aaa5b388d98da6480/src/boot">written in OCaml</a>. However, it was abandoned long ago, and the |
| only way to build a modern version of <code>rustc</code> is with a slightly less modern version.</p> |
| <p>This is exactly how <a href="https://github.com/rust-lang/rust/blob/HEAD/x.py"><code>./x.py</code></a> works: it downloads the current beta release of |
| <code>rustc</code>, then uses it to compile the new compiler.</p> |
| <p>Note that this documentation mostly covers user-facing information. See |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/bootstrap/README.md">bootstrap/README.md</a> to read about bootstrap internals.</p> |
| <h2 id="stages-of-bootstrapping"><a class="header" href="#stages-of-bootstrapping">Stages of bootstrapping</a></h2> |
| <h3 id="overview-1"><a class="header" href="#overview-1">Overview</a></h3> |
| <ul> |
| <li>Stage 0: the pre-compiled compiler and standard library</li> |
| <li>Stage 1: from current code, by an earlier compiler</li> |
| <li>Stage 2: the truly current compiler</li> |
| <li>Stage 3: the same-result test</li> |
| </ul> |
| <p>Compiling <code>rustc</code> is done in stages. Here's a diagram, adapted from Jynn |
| Nelson's <a href="https://www.youtube.com/watch?v=oUIjG-y4zaA">talk on bootstrapping</a> at RustConf 2022, with |
| detailed explanations below.</p> |
| <p>The <code>A</code>, <code>B</code>, <code>C</code>, and <code>D</code> show the ordering of the stages of bootstrapping. |
| <span style="background-color: lightblue; color: black">Blue</span> nodes are |
| downloaded, <span style="background-color: yellow; color: black">yellow</span> |
| nodes are built with the <code>stage0</code> compiler, and <span style="background-color: |
| lightgreen; color: black">green</span> nodes are built with the <code>stage1</code> |
| compiler.</p> |
| <pre class="mermaid">graph TD |
| s0c["stage0 compiler (1.86.0-beta.1)"]:::downloaded -->|A| s0l("stage0 std (1.86.0-beta.1)"):::downloaded; |
| s0c & s0l --- stepb[ ]:::empty; |
| stepb -->|B| s0ca["stage0 compiler artifacts (1.87.0-dev)"]:::with-s0c; |
| s0ca -->|copy| s1c["stage1 compiler (1.87.0-dev)"]:::with-s0c; |
| s1c -->|C| s1l("stage1 std (1.87.0-dev)"):::with-s1c; |
| s1c & s1l --- stepd[ ]:::empty; |
| stepd -->|D| s1ca["stage1 compiler artifacts (1.87.0-dev)"]:::with-s1c; |
| s1ca -->|copy| s2c["stage2 compiler"]:::with-s1c; |
| |
| classDef empty width:0px,height:0px; |
| classDef downloaded fill: lightblue; |
| classDef with-s0c fill: yellow; |
| classDef with-s1c fill: lightgreen; |
| </pre> |
| <h3 id="stage-0-the-pre-compiled-compiler"><a class="header" href="#stage-0-the-pre-compiled-compiler">Stage 0: the pre-compiled compiler</a></h3> |
| <p>The stage0 compiler is by default the very recent <em>beta</em> <code>rustc</code> compiler and its |
| associated dynamic libraries, which <code>./x.py</code> will download for you. (You can |
| also configure <code>./x.py</code> to change stage0 to something else.)</p> |
| <p>The precompiled stage0 compiler is then used only to compile <a href="https://github.com/rust-lang/rust/tree/HEAD/src/bootstrap"><code>src/bootstrap</code></a> and <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc"><code>compiler/rustc</code></a> |
| with precompiled stage0 std.</p> |
| <p>Note that to build the stage1 compiler we use the precompiled stage0 compiler and std. |
| Therefore, to use a compiler with a std that is freshly built from the tree, you need to |
| build the stage2 compiler.</p> |
| <p>There are two concepts at play here: a compiler (with its set of dependencies) and its |
| 'target' or 'object' libraries (<code>std</code> and <code>rustc</code>). Both are staged, but in a staggered manner.</p> |
| <h3 id="stage-1-from-current-code-by-an-earlier-compiler"><a class="header" href="#stage-1-from-current-code-by-an-earlier-compiler">Stage 1: from current code, by an earlier compiler</a></h3> |
| <p>The rustc source code is then compiled with the <code>stage0</code> compiler to produce the |
| <code>stage1</code> compiler.</p> |
| <h3 id="stage-2-the-truly-current-compiler"><a class="header" href="#stage-2-the-truly-current-compiler">Stage 2: the truly current compiler</a></h3> |
| <p>We then rebuild the compiler using <code>stage1</code> compiler with in-tree std to produce the <code>stage2</code> |
| compiler.</p> |
| <p>The <code>stage1</code> compiler itself was built by precompiled <code>stage0</code> compiler and std |
| and hence not by the source in your working directory. This means that the ABI |
| generated by the <code>stage0</code> compiler may not match the ABI that would have been made |
| by the <code>stage1</code> compiler, which can cause problems for dynamic libraries, tests |
| and tools using <code>rustc_private</code>.</p> |
| <p>Note that the <code>proc_macro</code> crate avoids this issue with a <code>C</code> FFI layer called |
| <code>proc_macro::bridge</code>, allowing it to be used with <code>stage1</code>.</p> |
| <p>The <code>stage2</code> compiler is the one distributed with <code>rustup</code> and all other install |
| methods. However, it takes a very long time to build because one must first |
| build the new compiler with an older compiler and then use that to build the new |
| compiler with itself.</p> |
| <p>For development, you usually only want to use <code>--stage 1</code> flag to build things. |
| See <a href="building/bootstrapping/../how-to-build-and-run.html#building-the-compiler">Building the compiler</a>.</p> |
| <h3 id="stage-3-the-same-result-test"><a class="header" href="#stage-3-the-same-result-test">Stage 3: the same-result test</a></h3> |
| <p>Stage 3 is optional. To sanity check our new compiler we can build the libraries |
| with the <code>stage2</code> compiler. The result ought to be identical to before, unless |
| something has broken.</p> |
| <h3 id="building-the-stages"><a class="header" href="#building-the-stages">Building the stages</a></h3> |
| <p>The script <a href="https://github.com/rust-lang/rust/blob/HEAD/x"><code>./x</code></a> tries to be helpful and pick the stage you most likely meant |
| for each subcommand. Here are some <code>x</code> commands with their default stages:</p> |
| <ul> |
| <li><code>check</code>: <code>--stage 1</code></li> |
| <li><code>clippy</code>: <code>--stage 1</code></li> |
| <li><code>doc</code>: <code>--stage 1</code></li> |
| <li><code>build</code>: <code>--stage 1</code></li> |
| <li><code>test</code>: <code>--stage 1</code></li> |
| <li><code>dist</code>: <code>--stage 2</code></li> |
| <li><code>install</code>: <code>--stage 2</code></li> |
| <li><code>bench</code>: <code>--stage 2</code></li> |
| </ul> |
| <p>You can always override the stage by passing <code>--stage N</code> explicitly.</p> |
| <p>For more information about stages, <a href="building/bootstrapping/what-bootstrapping-does.html#understanding-stages-of-bootstrap">see |
| below</a>.</p> |
| <h2 id="complications-of-bootstrapping"><a class="header" href="#complications-of-bootstrapping">Complications of bootstrapping</a></h2> |
| <p>Since the build system uses the current beta compiler to build a <code>stage1</code> |
| bootstrapping compiler, the compiler source code can't use some features until |
| they reach beta (because otherwise the beta compiler doesn't support them). On |
| the other hand, for <a href="building/bootstrapping/../../appendix/glossary.html#intrinsic">compiler intrinsics</a> and internal features, the |
| features <em>have</em> to be used. Additionally, the compiler makes heavy use of |
| <code>nightly</code> features (<code>#![feature(...)]</code>). How can we resolve this problem?</p> |
| <p>There are two methods used:</p> |
| <ol> |
| <li>The build system sets <code>--cfg bootstrap</code> when building with <code>stage0</code>, so we |
| can use <code>cfg(not(bootstrap))</code> to only use features when built with <code>stage1</code>. |
| Setting <code>--cfg bootstrap</code> in this way is used for features that were just |
| stabilized, which require <code>#![feature(...)]</code> when built with <code>stage0</code>, but |
| not for <code>stage1</code>.</li> |
| <li>The build system sets <code>RUSTC_BOOTSTRAP=1</code>. This special variable means to |
| <em>break the stability guarantees</em> of Rust: allowing use of <code>#![feature(...)]</code> |
| with a compiler that's not <code>nightly</code>. <em>Setting <code>RUSTC_BOOTSTRAP=1</code> should |
| never be used except when bootstrapping the compiler.</em></li> |
| </ol> |
| <h2 id="understanding-stages-of-bootstrap"><a class="header" href="#understanding-stages-of-bootstrap">Understanding stages of bootstrap</a></h2> |
| <h3 id="overview-2"><a class="header" href="#overview-2">Overview</a></h3> |
| <p>This is a detailed look into the separate bootstrap stages.</p> |
| <p>The convention <code>./x</code> uses is that:</p> |
| <ul> |
| <li>A <code>--stage N</code> flag means to run the stage N compiler (<code>stageN/rustc</code>).</li> |
| <li>A "stage N artifact" is a build artifact that is <em>produced</em> by the stage N |
| compiler.</li> |
| <li>The stage N+1 compiler is assembled from stage N <em>artifacts</em>. This process is |
| called <em>uplifting</em>.</li> |
| </ul> |
| <h4 id="build-artifacts"><a class="header" href="#build-artifacts">Build artifacts</a></h4> |
| <p>Anything you can build with <code>./x</code> is a <em>build artifact</em>. Build artifacts |
| include, but are not limited to:</p> |
| <ul> |
| <li>binaries, like <code>stage0-rustc/rustc-main</code></li> |
| <li>shared objects, like <code>stage0-sysroot/rustlib/libstd-6fae108520cf72fe.so</code></li> |
| <li><a href="building/bootstrapping/../../serialization.html">rlib</a> files, like <code>stage0-sysroot/rustlib/libstd-6fae108520cf72fe.rlib</code></li> |
| <li>HTML files generated by rustdoc, like <code>doc/std</code></li> |
| </ul> |
| <h4 id="examples"><a class="header" href="#examples">Examples</a></h4> |
| <ul> |
| <li><code>./x test tests/ui</code> means to build the <code>stage1</code> compiler and run <code>compiletest</code> |
| on it. If you're working on the compiler, this is normally the test command |
| you want.</li> |
| <li><code>./x test --stage 0 library/std</code> means to run tests on the standard library |
| without building <code>rustc</code> from source ('build with <code>stage0</code>, then test the |
| artifacts'). If you're working on the standard library, this is normally the |
| test command you want.</li> |
| <li><code>./x build --stage 0</code> means to build with the stage0 <code>rustc</code>.</li> |
| <li><code>./x doc --stage 1</code> means to document using the stage0 <code>rustdoc</code>.</li> |
| </ul> |
| <h4 id="examples-of-what-not-to-do"><a class="header" href="#examples-of-what-not-to-do">Examples of what <em>not</em> to do</a></h4> |
| <ul> |
| <li><code>./x test --stage 0 tests/ui</code> is not useful: it runs tests on the <em>beta</em> |
| compiler and doesn't build <code>rustc</code> from source. Use <code>test tests/ui</code> instead, |
| which builds <code>stage1</code> from source.</li> |
| <li><code>./x test --stage 0 compiler/rustc</code> builds the compiler but runs no tests: |
| it's running <code>cargo test -p rustc</code>, but <code>cargo</code> doesn't understand Rust's |
| tests. You shouldn't need to use this, use <code>test</code> instead (without arguments).</li> |
| <li><code>./x build --stage 0 compiler/rustc</code> builds the compiler, but does not build |
| <code>libstd</code> or even <code>libcore</code>. Most of the time, you'll want <code>./x build library</code> |
| instead, which allows compiling programs without needing to define lang items.</li> |
| </ul> |
| <h3 id="building-vs-running"><a class="header" href="#building-vs-running">Building vs. running</a></h3> |
| <p>In short, <em>stage 0 uses the <code>stage0</code> compiler to create <code>stage0</code> artifacts which |
| will later be uplifted to be the stage1 compiler</em>.</p> |
| <p>In each stage besides 0, two major steps are performed:</p> |
| <ol> |
| <li><code>std</code> is compiled by the stage N compiler.</li> |
| <li>That <code>std</code> is linked to programs built by the stage N compiler, including the |
| stage N artifacts (stage N+1 compiler).</li> |
| </ol> |
| <p>This is somewhat intuitive if one thinks of the stage N artifacts as "just" |
| another program we are building with the stage N compiler: <code>build --stage N compiler/rustc</code> is linking the stage N artifacts to the <code>std</code> built by the stage |
| N compiler.</p> |
| <h3 id="stages-and-std"><a class="header" href="#stages-and-std">Stages and <code>std</code></a></h3> |
| <p>Note that there are two <code>std</code> libraries in play here:</p> |
| <ol> |
| <li>The library <em>linked</em> to <code>stageN/rustc</code>, which was built by stage N-1 (stage |
| N-1 <code>std</code>)</li> |
| <li>The library <em>used to compile programs</em> with <code>stageN/rustc</code>, which was built |
| by stage N (stage N <code>std</code>).</li> |
| </ol> |
| <p>Stage N <code>std</code> is pretty much necessary for any useful work with the stage N |
| compiler. Without it, you can only compile programs with <code>#![no_core]</code> -- not |
| terribly useful!</p> |
| <p>The reason these need to be different is because they aren't necessarily |
| ABI-compatible: there could be new layout optimizations, changes to <code>MIR</code>, or |
| other changes to Rust metadata on <code>nightly</code> that aren't present in beta.</p> |
| <p>This is also where <code>--keep-stage 1 library/std</code> comes into play. Since most |
| changes to the compiler don't actually change the ABI, once you've produced a |
| <code>std</code> in <code>stage1</code>, you can probably just reuse it with a different compiler. If |
| the ABI hasn't changed, you're good to go, no need to spend time recompiling |
| that <code>std</code>. The flag <code>--keep-stage</code> simply instructs the build script to assumes |
| the previous compile is fine and copies those artifacts into the appropriate |
| place, skipping the <code>cargo</code> invocation.</p> |
| <h3 id="cross-compiling-rustc"><a class="header" href="#cross-compiling-rustc">Cross-compiling rustc</a></h3> |
| <p><em>Cross-compiling</em> is the process of compiling code that will run on another |
| architecture. For instance, you might want to build an ARM version of rustc |
| using an x86 machine. Building <code>stage2</code> <code>std</code> is different when you are |
| cross-compiling.</p> |
| <p>This is because <code>./x</code> uses the following logic: if <code>HOST</code> and <code>TARGET</code> are the |
| same, it will reuse <code>stage1</code> <code>std</code> for <code>stage2</code>! This is sound because <code>stage1</code> |
| <code>std</code> was compiled with the <code>stage1</code> compiler, i.e. a compiler using the source |
| code you currently have checked out. So it should be identical (and therefore |
| ABI-compatible) to the <code>std</code> that <code>stage2/rustc</code> would compile.</p> |
| <p>However, when cross-compiling, <code>stage1</code> <code>std</code> will only run on the host. So the |
| <code>stage2</code> compiler has to recompile <code>std</code> for the target.</p> |
| <p>(See in the table how <code>stage2</code> only builds non-host <code>std</code> targets).</p> |
| <h3 id="what-is-a-sysroot"><a class="header" href="#what-is-a-sysroot">What is a 'sysroot'?</a></h3> |
| <p>When you build a project with <code>cargo</code>, the build artifacts for dependencies are |
| normally stored in <code>target/debug/deps</code>. This only contains dependencies <code>cargo</code> |
| knows about; in particular, it doesn't have the standard library. Where do <code>std</code> |
| or <code>proc_macro</code> come from? They come from the <strong>sysroot</strong>, the root of a number |
| of directories where the compiler loads build artifacts at runtime. The |
| <code>sysroot</code> doesn't just store the standard library, though - it includes anything |
| that needs to be loaded at runtime. That includes (but is not limited to):</p> |
| <ul> |
| <li>Libraries <code>libstd</code>/<code>libtest</code>/<code>libproc_macro</code>.</li> |
| <li>Compiler crates themselves, when using <code>rustc_private</code>. In-tree these are |
| always present; out of tree, you need to install <code>rustc-dev</code> with <code>rustup</code>.</li> |
| <li>Shared object file <code>libLLVM.so</code> for the LLVM project. In-tree this is either |
| built from source or downloaded from CI; out-of-tree, you need to install |
| <code>llvm-tools-preview</code> with <code>rustup</code>.</li> |
| </ul> |
| <p>All the artifacts listed so far are <em>compiler</em> runtime dependencies. You can see |
| them with <code>rustc --print sysroot</code>:</p> |
| <pre><code>$ ls $(rustc --print sysroot)/lib |
| libchalk_derive-0685d79833dc9b2b.so libstd-25c6acf8063a3802.so |
| libLLVM-11-rust-1.50.0-nightly.so libtest-57470d2aa8f7aa83.so |
| librustc_driver-4f0cc9f50e53f0ba.so libtracing_attributes-e4be92c35ab2a33b.so |
| librustc_macros-5f0ec4a119c6ac86.so rustlib |
| </code></pre> |
| <p>There are also runtime dependencies for the standard library! These are in |
| <code>lib/rustlib/</code>, not <code>lib/</code> directly.</p> |
| <pre><code>$ ls $(rustc --print sysroot)/lib/rustlib/x86_64-unknown-linux-gnu/lib | head -n 5 |
| libaddr2line-6c8e02b8fedc1e5f.rlib |
| libadler-9ef2480568df55af.rlib |
| liballoc-9c4002b5f79ba0e1.rlib |
| libcfg_if-512eb53291f6de7e.rlib |
| libcompiler_builtins-ef2408da76957905.rlib |
| </code></pre> |
| <p>Directory <code>lib/rustlib/</code> includes libraries like <code>hashbrown</code> and <code>cfg_if</code>, which |
| are not part of the public API of the standard library, but are used to |
| implement it. Also <code>lib/rustlib/</code> is part of the search path for linkers, but |
| <code>lib</code> will never be part of the search path.</p> |
| <h4 id="-z-force-unstable-if-unmarked"><a class="header" href="#-z-force-unstable-if-unmarked"><code>-Z force-unstable-if-unmarked</code></a></h4> |
| <p>Since <code>lib/rustlib/</code> is part of the search path we have to be careful about |
| which crates are included in it. In particular, all crates except for the |
| standard library are built with the flag <code>-Z force-unstable-if-unmarked</code>, which |
| means that you have to use <code>#![feature(rustc_private)]</code> in order to load it (as |
| opposed to the standard library, which is always available).</p> |
| <p>The <code>-Z force-unstable-if-unmarked</code> flag has a variety of purposes to help |
| enforce that the correct crates are marked as <code>unstable</code>. It was introduced |
| primarily to allow rustc and the standard library to link to arbitrary crates on |
| crates.io which do not themselves use <code>staged_api</code>. <code>rustc</code> also relies on this |
| flag to mark all of its crates as <code>unstable</code> with the <code>rustc_private</code> feature so |
| that each crate does not need to be carefully marked with <code>unstable</code>.</p> |
| <p>This flag is automatically applied to all of <code>rustc</code> and the standard library by |
| the bootstrap scripts. This is needed because the compiler and all of its |
| dependencies are shipped in <code>sysroot</code> to all users.</p> |
| <p>This flag has the following effects:</p> |
| <ul> |
| <li>Marks the crate as "<code>unstable</code>" with the <code>rustc_private</code> feature if it is not |
| itself marked as <code>stable</code> or <code>unstable</code>.</li> |
| <li>Allows these crates to access other forced-unstable crates without any need |
| for attributes. Normally a crate would need a <code>#![feature(rustc_private)]</code> |
| attribute to use other <code>unstable</code> crates. However, that would make it |
| impossible for a crate from crates.io to access its own dependencies since |
| that crate won't have a <code>feature(rustc_private)</code> attribute, but <em>everything</em> |
| is compiled with <code>-Z force-unstable-if-unmarked</code>.</li> |
| </ul> |
| <p>Code which does not use <code>-Z force-unstable-if-unmarked</code> should include the |
| <code>#![feature(rustc_private)]</code> crate attribute to access these forced-unstable |
| crates. This is needed for things which link <code>rustc</code> its self, such as <code>MIRI</code> or |
| <code>clippy</code>.</p> |
| <p>You can find more discussion about sysroots in:</p> |
| <ul> |
| <li>The <a href="https://github.com/rust-lang/rust/pull/76728">rustdoc PR</a> explaining why it uses <code>extern crate</code> for dependencies loaded |
| from <code>sysroot</code></li> |
| <li><a href="https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp/topic/deps.20in.20sysroot/">Discussions about sysroot on |
| Zulip</a></li> |
| <li><a href="https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp/topic/How.20to.20create.20an.20executable.20accessing.20.60rustc_private.60.3F">Discussions about building rustdoc out of |
| tree</a></li> |
| </ul> |
| <h2 id="passing-flags-to-commands-invoked-by-bootstrap"><a class="header" href="#passing-flags-to-commands-invoked-by-bootstrap">Passing flags to commands invoked by <code>bootstrap</code></a></h2> |
| <p>Conveniently <code>./x</code> allows you to pass stage-specific flags to <code>rustc</code> and |
| <code>cargo</code> when bootstrapping. The <code>RUSTFLAGS_BOOTSTRAP</code> environment variable is |
| passed as <code>RUSTFLAGS</code> to the bootstrap stage (<code>stage0</code>), and |
| <code>RUSTFLAGS_NOT_BOOTSTRAP</code> is passed when building artifacts for later stages. |
| <code>RUSTFLAGS</code> will work, but also affects the build of <code>bootstrap</code> itself, so it |
| will be rare to want to use it. Finally, <code>MAGIC_EXTRA_RUSTFLAGS</code> bypasses the |
| <code>cargo</code> cache to pass flags to rustc without recompiling all dependencies.</p> |
| <ul> |
| <li><code>RUSTDOCFLAGS</code>, <code>RUSTDOCFLAGS_BOOTSTRAP</code> and <code>RUSTDOCFLAGS_NOT_BOOTSTRAP</code> are |
| analogous to <code>RUSTFLAGS</code>, but for <code>rustdoc</code>.</li> |
| <li><code>CARGOFLAGS</code> will pass arguments to cargo itself (e.g. <code>--timings</code>). |
| <code>CARGOFLAGS_BOOTSTRAP</code> and <code>CARGOFLAGS_NOT_BOOTSTRAP</code> work analogously to |
| <code>RUSTFLAGS_BOOTSTRAP</code>.</li> |
| <li><code>--test-args</code> will pass arguments through to the test runner. For <code>tests/ui</code>, |
| this is <code>compiletest</code>. For unit tests and doc tests this is the <code>libtest</code> |
| runner.</li> |
| </ul> |
| <p>Most test runners accept <code>--help</code>, |
| which you can use to find out the options accepted by the runner.</p> |
| <h2 id="environment-variables"><a class="header" href="#environment-variables">Environment Variables</a></h2> |
| <p>During bootstrapping, there are a bunch of compiler-internal environment |
| variables that are used. If you are trying to run an intermediate version of |
| <code>rustc</code>, sometimes you may need to set some of these environment variables |
| manually. Otherwise, you get an error like the following:</p> |
| <pre><code class="language-text">thread 'main' panicked at 'RUSTC_STAGE was not set: NotPresent', library/core/src/result.rs:1165:5 |
| </code></pre> |
| <p>If <code>./stageN/bin/rustc</code> gives an error about environment variables, that usually |
| means something is quite wrong -- such as you're trying to compile <code>rustc</code> or |
| <code>std</code> or something which depends on environment variables. In the unlikely case |
| that you actually need to invoke <code>rustc</code> in such a situation, you can tell the |
| bootstrap shim to print all <code>env</code> variables by adding <code>-vvv</code> to your <code>x</code> |
| command.</p> |
| <p>Finally, bootstrap makes use of the <a href="https://github.com/rust-lang/cc-rs">cc-rs crate</a> which has <a href="https://docs.rs/cc/latest/cc/#external-configuration-via-environment-variables">its own |
| method</a> of configuring <code>C</code> compilers and <code>C</code> flags via environment |
| variables.</p> |
| <h2 id="clarification-of-build-commands-stdout"><a class="header" href="#clarification-of-build-commands-stdout">Clarification of build command's <code>stdout</code></a></h2> |
| <p>In this part, we will investigate the build command's <code>stdout</code> in an action |
| (similar, but more detailed and complete documentation compare to topic above). |
| When you execute <code>x build --dry-run</code> command, the build output will be something |
| like the following:</p> |
| <pre><code class="language-text">Building stage0 library artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu) |
| Copying stage0 library from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) |
| Building stage0 compiler artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu) |
| Copying stage0 rustc from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) |
| Assembling stage1 compiler (x86_64-unknown-linux-gnu) |
| Building stage1 library artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu) |
| Copying stage1 library from stage1 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) |
| Building stage1 tool rust-analyzer-proc-macro-srv (x86_64-unknown-linux-gnu) |
| Building rustdoc for stage1 (x86_64-unknown-linux-gnu) |
| </code></pre> |
| <h3 id="building-stage0-stdcompiler-artifacts"><a class="header" href="#building-stage0-stdcompiler-artifacts">Building stage0 {std,compiler} artifacts</a></h3> |
| <p>These steps use the provided (downloaded, usually) compiler to compile the local |
| Rust source into libraries we can use.</p> |
| <h3 id="copying-stage0-stdrustc"><a class="header" href="#copying-stage0-stdrustc">Copying stage0 {std,rustc}</a></h3> |
| <p>This copies the library and compiler artifacts from <code>cargo</code> into |
| <code>stage0-sysroot/lib/rustlib/{target-triple}/lib</code></p> |
| <h3 id="assembling-stage1-compiler"><a class="header" href="#assembling-stage1-compiler">Assembling stage1 compiler</a></h3> |
| <p>This copies the libraries we built in "building <code>stage0</code> ... artifacts" into the |
| <code>stage1</code> compiler's <code>lib/</code> directory. These are the host libraries that the |
| compiler itself uses to run. These aren't actually used by artifacts the new |
| compiler generates. This step also copies the <code>rustc</code> and <code>rustdoc</code> binaries we |
| generated into <code>build/$HOST/stage/bin</code>.</p> |
| <p>The <code>stage1/bin/rustc</code> is a fully functional compiler built with stage0 (precompiled) compiler and std. |
| To use a compiler built entirely from source with the in-tree compiler and std, you need to build the |
| stage2 compiler, which is compiled using the stage1 (in-tree) compiler and std.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="how-bootstrap-does-it"><a class="header" href="#how-bootstrap-does-it">How Bootstrap does it</a></h1> |
| <p>The core concept in Bootstrap is a build <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a>, which are chained together |
| by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/struct.Builder.html#method.ensure"><code>Builder::ensure</code></a>. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/struct.Builder.html#method.ensure"><code>Builder::ensure</code></a> takes a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a> as input, and runs |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a> if and only if it has not already been run. Let's take a closer |
| look at <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a>.</p> |
| <h2 id="synopsis-of-step"><a class="header" href="#synopsis-of-step">Synopsis of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a></a></h2> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a> represents a granular collection of actions involved in the process |
| of producing some artifact. It can be thought of like a rule in Makefiles. |
| The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a> trait is defined as:</p> |
| <pre><code class="language-rs no_run">pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { |
| type Output: Clone; |
| |
| const DEFAULT: bool = false; |
| const ONLY_HOSTS: bool = false; |
| |
| // Required methods |
| fn run(self, builder: &Builder<'_>) -> Self::Output; |
| fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; |
| |
| // Provided method |
| fn make_run(_run: RunConfig<'_>) { ... } |
| } |
| </code></pre> |
| <ul> |
| <li><code>run</code> is the function that is responsible for doing the work. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/struct.Builder.html#method.ensure"><code>Builder::ensure</code></a> invokes <code>run</code>.</li> |
| <li><code>should_run</code> is the command-line interface, which determines if an invocation |
| such as <code>x build foo</code> should run a given <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a>. In a "default" context |
| where no paths are provided, then <code>make_run</code> is called directly.</li> |
| <li><code>make_run</code> is invoked only for things directly asked via the CLI and not |
| for steps which are dependencies of other steps.</li> |
| </ul> |
| <h2 id="the-entry-points"><a class="header" href="#the-entry-points">The entry points</a></h2> |
| <p>There's a couple of preliminary steps before core Bootstrap code is reached:</p> |
| <ol> |
| <li>Shell script or <code>make</code>: <a href="https://github.com/rust-lang/rust/blob/HEAD/x"><code>./x</code></a> or <a href="https://github.com/rust-lang/rust/blob/HEAD/x.ps1"><code>./x.ps1</code></a> or <code>make</code></li> |
| <li>Convenience wrapper script: <a href="https://github.com/rust-lang/rust/blob/HEAD/x.py"><code>x.py</code></a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/src/bootstrap/bootstrap.py"><code>src/bootstrap/bootstrap.py</code></a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/src/bootstrap/src/bin/main.rs"><code>src/bootstrap/src/bin/main.rs</code></a></li> |
| </ol> |
| <p>See <a href="https://github.com/rust-lang/rust/blob/HEAD/src/bootstrap/README.md">src/bootstrap/README.md</a> |
| for a more specific description of the implementation details.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="writing-tools-in-bootstrap"><a class="header" href="#writing-tools-in-bootstrap">Writing tools in Bootstrap</a></h1> |
| <p>There are three types of tools you can write in bootstrap:</p> |
| <ul> |
| <li> |
| <p><strong><code>Mode::ToolBootstrap</code></strong></p> |
| <p>Use this for tools that don’t need anything from the in-tree compiler and can run with the stage0 <code>rustc</code>. |
| The output is placed in the "bootstrap-tools" directory. |
| This mode is for general-purpose tools built entirely with the stage0 compiler, |
| including target libraries, and it only works for stage 0.</p> |
| </li> |
| <li> |
| <p><strong><code>Mode::ToolStd</code></strong></p> |
| <p>Use this for tools that rely on the locally built std. |
| The output goes into the "stageN-tools" directory. |
| This mode is rarely used, mainly for <code>compiletest</code> which requires <code>libtest</code>.</p> |
| </li> |
| <li> |
| <p><strong><code>Mode::ToolRustcPrivate</code></strong></p> |
| <p>Use this for tools that use the <code>rustc_private</code> mechanism, |
| and thus depend on the locally built <code>rustc</code> and its rlib artifacts. |
| This is more complex than the other modes, |
| because the tool must be built with the same compiler used for <code>rustc</code>, |
| and placed in the "stageN-tools" directory. |
| When you choose <code>Mode::ToolRustcPrivate</code>, |
| <code>ToolBuild</code> implementation takes care of this automatically. |
| If you need to use the builder’s compiler for something specific, |
| you can get it from <code>ToolBuildResult</code>, which is returned by the tool's <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a>.</p> |
| </li> |
| </ul> |
| <p>Regardless of the tool type, |
| you must return <code>ToolBuildResult</code> from the tool’s <a href="https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html"><code>Step</code></a> implementation, |
| and use <code>ToolBuild</code> inside it.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="debugging-bootstrap"><a class="header" href="#debugging-bootstrap">Debugging bootstrap</a></h1> |
| <p>There are two main ways of debugging (and profiling bootstrap). The first is through println logging, and the second is through the <code>tracing</code> feature.</p> |
| <h2 id="println-logging"><a class="header" href="#println-logging"><code>println</code> logging</a></h2> |
| <p>Bootstrap has extensive unstructured logging. Most of it is gated behind the <code>--verbose</code> flag (pass <code>-vv</code> for even more detail).</p> |
| <p>If you want to see verbose output of executed Cargo commands and other kinds of detailed logs, pass <code>-v</code> or <code>-vv</code> when invoking bootstrap. Note that the logs are unstructured and may be overwhelming.</p> |
| <pre><code>$ ./x dist rustc --dry-run -vv |
| learning about cargo |
| running: RUSTC_BOOTSTRAP="1" "/home/jyn/src/rust2/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "metadata" "--format-version" "1" "--no-deps" "--manifest-path" "/home/jyn/src/rust2/Cargo.toml" (failure_mode=Exit) (created at src/bootstrap/src/core/metadata.rs:81:25, executed at src/bootstrap/src/core/metadata.rs:92:50) |
| running: RUSTC_BOOTSTRAP="1" "/home/jyn/src/rust2/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "metadata" "--format-version" "1" "--no-deps" "--manifest-path" "/home/jyn/src/rust2/library/Cargo.toml" (failure_mode=Exit) (created at src/bootstrap/src/core/metadata.rs:81:25, executed at src/bootstrap/src/core/metadata.rs:92:50) |
| ... |
| </code></pre> |
| <h2 id="tracing-in-bootstrap"><a class="header" href="#tracing-in-bootstrap"><code>tracing</code> in bootstrap</a></h2> |
| <p>Bootstrap has a conditional <code>tracing</code> feature, which provides the following features:</p> |
| <ul> |
| <li>It enables structured logging using <a href="https://docs.rs/tracing/0.1.41/tracing/index.html"><code>tracing</code></a> events and spans.</li> |
| <li>It generates a <a href="https://www.chromium.org/developers/how-tos/trace-event-profiling-tool/">Chrome trace file</a> that can be used to visualize the hierarchy and durations of executed steps and commands. |
| <ul> |
| <li>You can open the generated <code>chrome-trace.json</code> file using Chrome, on the <code>chrome://tracing</code> tab, or e.g. using <a href="https://ui.perfetto.dev/">Perfetto</a>.</li> |
| </ul> |
| </li> |
| <li>It generates <a href="https://graphviz.org/doc/info/lang.html">GraphViz</a> graphs that visualize the dependencies between executed steps. |
| <ul> |
| <li>You can open the generated <code>step-graph-*.dot</code> file using e.g. <a href="https://github.com/jrfonseca/xdot.py">xdot</a> to visualize the step graph, or use e.g. <code>dot -Tsvg</code> to convert the GraphViz file to an SVG file.</li> |
| </ul> |
| </li> |
| <li>It generates a command execution summary, which shows which commands were executed, how many of their executions were cached, and what commands were the slowest to run. |
| <ul> |
| <li>The generated <code>command-stats.txt</code> file is in a simple human-readable format.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>The structured logs will be written to standard error output (<code>stderr</code>), while the other outputs will be stored in files in the <code><build-dir>/bootstrap-trace/<pid></code> directory. For convenience, bootstrap will also create a symlink to the latest generated trace output directory at <code><build-dir>/bootstrap-trace/latest</code>.</p> |
| <blockquote> |
| <p>Note that if you execute bootstrap with <code>--dry-run</code>, the tracing output directory might change. Bootstrap will always print a path where the tracing output files were stored at the end of its execution.</p> |
| </blockquote> |
| <h3 id="enabling-tracing-output"><a class="header" href="#enabling-tracing-output">Enabling <code>tracing</code> output</a></h3> |
| <p>To enable the conditional <code>tracing</code> feature, run bootstrap with the <code>BOOTSTRAP_TRACING</code> environment variable.</p> |
| <pre><code class="language-bash">$ BOOTSTRAP_TRACING=trace ./x build library --stage 1 |
| </code></pre> |
| <p>Example output<sup class="footnote-reference" id="fr-unstable-1"><a href="#footnote-unstable">1</a></sup>:</p> |
| <pre><code>$ BOOTSTRAP_TRACING=trace ./x build library --stage 1 --dry-run |
| Building bootstrap |
| Finished `dev` profile [unoptimized] target(s) in 0.05s |
| 15:56:52.477 INFO > tool::LibcxxVersionTool {target: x86_64-unknown-linux-gnu} (builder/mod.rs:1715) |
| 15:56:52.575 INFO > compile::Assemble {target_compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }} (builder/mod.rs:1715) |
| 15:56:52.575 INFO > tool::Compiletest {compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, target: x86_64-unknown-linux-gnu} (builder/mod.rs:1715) |
| 15:56:52.576 INFO > tool::ToolBuild {build_compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, target: x86_64-unknown-linux-gnu, tool: "compiletest", path: "src/tools/compiletest", mode: ToolBootstrap, source_type: InTree, extra_features: [], allow_features: "internal_output_capture", cargo_args: [], artifact_kind: Binary} (builder/mod.rs:1715) |
| 15:56:52.576 INFO > builder::Libdir {compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, target: x86_64-unknown-linux-gnu} (builder/mod.rs:1715) |
| 15:56:52.576 INFO > compile::Sysroot {compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, force_recompile: false} (builder/mod.rs:1715) |
| 15:56:52.578 INFO > compile::Assemble {target_compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }} (builder/mod.rs:1715) |
| 15:56:52.578 INFO > tool::Compiletest {compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, target: x86_64-unknown-linux-gnu} (builder/mod.rs:1715) |
| 15:56:52.578 INFO > tool::ToolBuild {build_compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, target: x86_64-unknown-linux-gnu, tool: "compiletest", path: "src/tools/compiletest", mode: ToolBootstrap, source_type: InTree, extra_features: [], allow_features: "internal_output_capture", cargo_args: [], artifact_kind: Binary} (builder/mod.rs:1715) |
| 15:56:52.578 INFO > builder::Libdir {compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, target: x86_64-unknown-linux-gnu} (builder/mod.rs:1715) |
| 15:56:52.578 INFO > compile::Sysroot {compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu, forced_compiler: false }, force_recompile: false} (builder/mod.rs:1715) |
| Finished `release` profile [optimized] target(s) in 0.11s |
| Tracing/profiling output has been written to <src-root>/build/bootstrap-trace/latest |
| Build completed successfully in 0:00:00 |
| </code></pre> |
| <h4 id="controlling-tracing-output"><a class="header" href="#controlling-tracing-output">Controlling tracing output</a></h4> |
| <p>The environment variable <code>BOOTSTRAP_TRACING</code> accepts a <a href="https://docs.rs/tracing-subscriber/0.3.19/tracing_subscriber/filter/struct.EnvFilter.html"><code>tracing_subscriber</code> filter</a>. If you set <code>BOOTSTRAP_TRACING=trace</code>, you will enable all logs, but that can be overwhelming. You can thus use the filter to reduce the amount of data logged.</p> |
| <p>There are two orthogonal ways to control which kind of tracing logs you want:</p> |
| <ol> |
| <li>You can specify the log <strong>level</strong>, e.g. <code>debug</code> or <code>trace</code>. |
| <ul> |
| <li>If you select a level, all events/spans with an equal or higher priority level will be shown.</li> |
| </ul> |
| </li> |
| <li>You can also control the log <strong>target</strong>, e.g. <code>bootstrap</code> or <code>bootstrap::core::config</code> or a custom target like <code>CONFIG_HANDLING</code> or <code>STEP</code>. |
| <ul> |
| <li>Custom targets are used to limit what kinds of spans you are interested in, as the <code>BOOTSTRAP_TRACING=trace</code> output can be quite verbose. Currently, you can use the following custom targets: |
| <ul> |
| <li><code>CONFIG_HANDLING</code>: show spans related to config handling.</li> |
| <li><code>STEP</code>: show all executed steps. Executed commands have <code>info</code> event level.</li> |
| <li><code>COMMAND</code>: show all executed commands. Executed commands have <code>trace</code> event level.</li> |
| <li><code>IO</code>: show performed I/O operations. Executed commands have <code>trace</code> event level. |
| <ul> |
| <li>Note that many I/O are currently not being traced.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ol> |
| <p>You can of course combine them (custom target logs are typically gated behind <code>TRACE</code> log level additionally):</p> |
| <pre><code class="language-bash">$ BOOTSTRAP_TRACING=CONFIG_HANDLING=trace,STEP=info,COMMAND=trace ./x build library --stage 1 |
| </code></pre> |
| <p>Note that the level that you specify using <code>BOOTSTRAP_TRACING</code> also has an effect on the spans that will be recorded in the Chrome trace file.</p> |
| <h5 id="fixme96176-specific-tracing-for-compiler-vs-compiler_for"><a class="header" href="#fixme96176-specific-tracing-for-compiler-vs-compiler_for">FIXME(#96176): specific tracing for <code>compiler()</code> vs <code>compiler_for()</code></a></h5> |
| <p>The additional targets <code>COMPILER</code> and <code>COMPILER_FOR</code> are used to help trace what |
| <code>builder.compiler()</code> and <code>builder.compiler_for()</code> does. They should be removed |
| if <a href="https://github.com/rust-lang/rust/issues/96176">#96176</a> is resolved.</p> |
| <h3 id="using-tracing-in-bootstrap"><a class="header" href="#using-tracing-in-bootstrap">Using <code>tracing</code> in bootstrap</a></h3> |
| <p>Both <code>tracing::*</code> macros and the <code>tracing::instrument</code> proc-macro attribute need to be gated behind <code>tracing</code> feature. Examples:</p> |
| <pre><code class="language-rs">#[cfg(feature = "tracing")] |
| use tracing::instrument; |
| |
| struct Foo; |
| |
| impl Step for Foo { |
| type Output = (); |
| |
| #[cfg_attr(feature = "tracing", instrument(level = "trace", name = "Foo::should_run", skip_all))] |
| fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { |
| trace!(?run, "entered Foo::should_run"); |
| |
| todo!() |
| } |
| |
| fn run(self, builder: &Builder<'_>) -> Self::Output { |
| trace!(?run, "entered Foo::run"); |
| |
| todo!() |
| } |
| } |
| </code></pre> |
| <p>For <code>#[instrument]</code>, it's recommended to:</p> |
| <ul> |
| <li>Gate it behind <code>trace</code> level for fine-granularity, possibly <code>debug</code> level for core functions.</li> |
| <li>Explicitly pick an instrumentation name via <code>name = ".."</code> to distinguish between e.g. <code>run</code> of different steps.</li> |
| <li>Take care to not cause diverging behavior via tracing, e.g. building extra things only when tracing infra is enabled.</li> |
| </ul> |
| <h3 id="rust-analyzer-integration"><a class="header" href="#rust-analyzer-integration">rust-analyzer integration?</a></h3> |
| <p>Unfortunately, because bootstrap is a <code>rust-analyzer.linkedProjects</code>, you can't ask r-a to check/build bootstrap itself with <code>tracing</code> feature enabled to get relevant completions, due to lack of support as described in <a href="https://github.com/rust-lang/rust-analyzer/issues/8521">https://github.com/rust-lang/rust-analyzer/issues/8521</a>.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-unstable"> |
| <p>This output is always subject to further changes. <a href="#fr-unstable-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="cfgbootstrap-in-compiler-dependencies"><a class="header" href="#cfgbootstrap-in-compiler-dependencies"><code>cfg(bootstrap)</code> in compiler dependencies</a></h1> |
| <p>The rust compiler uses some external crates that can run into cyclic dependencies with the compiler itself: the compiler needs an updated crate to build, but the crate needs an updated compiler. This page describes how <code>#[cfg(bootstrap)]</code> can be used to break this cycle.</p> |
| <h2 id="enabling-cfgbootstrap"><a class="header" href="#enabling-cfgbootstrap">Enabling <code>#[cfg(bootstrap)]</code></a></h2> |
| <p>Usually the use of <code>#[cfg(bootstrap)]</code> in an external crate causes a warning:</p> |
| <pre><code>warning: unexpected `cfg` condition name: `bootstrap` |
| --> src/main.rs:1:7 |
| | |
| 1 | #[cfg(bootstrap)] |
| | ^^^^^^^^^ |
| | |
| = help: expected names are: `docsrs`, `feature`, and `test` and 31 more |
| = help: consider using a Cargo feature instead |
| = help: or consider adding in `Cargo.toml` the `check-cfg` lint config for the lint: |
| [lints.rust] |
| unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bootstrap)'] } |
| = help: or consider adding `println!("cargo::rustc-check-cfg=cfg(bootstrap)");` to the top of the `build.rs` |
| = note: see <https://doc.rust-lang.org/nightly/rustc/check-cfg/cargo-specifics.html> for more information about checking conditional configuration |
| = note: `#[warn(unexpected_cfgs)]` on by default |
| </code></pre> |
| <p>This warning can be silenced by adding these lines to the project's <code>Cargo.toml</code>:</p> |
| <pre><code class="language-toml">[lints.rust] |
| unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bootstrap)'] } |
| </code></pre> |
| <p>Now <code>#[cfg(bootstrap)]</code> can be used in the crate just like it can be in the compiler: when the bootstrap compiler is used, code annotated with <code>#[cfg(bootstrap)]</code> is compiled, otherwise code annotated with <code>#[cfg(not(bootstrap))]</code> is compiled.</p> |
| <h2 id="the-update-dance"><a class="header" href="#the-update-dance">The update dance</a></h2> |
| <p>As a concrete example we'll use a change where the <code>#[naked]</code> attribute was made into an unsafe attribute, which caused a cyclic dependency with the <code>compiler-builtins</code> crate.</p> |
| <h3 id="step-1-accept-the-new-behavior-in-the-compiler-139797"><a class="header" href="#step-1-accept-the-new-behavior-in-the-compiler-139797">Step 1: accept the new behavior in the compiler (<a href="https://github.com/rust-lang/rust/pull/139797">#139797</a>)</a></h3> |
| <p>In this example it is possible to accept both the old and new behavior at the same time by disabling an error.</p> |
| <h3 id="step-2-update-the-crate-821"><a class="header" href="#step-2-update-the-crate-821">Step 2: update the crate (<a href="https://github.com/rust-lang/compiler-builtins/pull/821">#821</a>)</a></h3> |
| <p>Now in the crate, use <code>#[cfg(bootstrap)]</code> to use the old behavior, or <code>#[cfg(not(bootstrap))]</code> to use the new behavior.</p> |
| <h3 id="step-3-update-the-crate-version-used-by-the-compiler-139934"><a class="header" href="#step-3-update-the-crate-version-used-by-the-compiler-139934">Step 3: update the crate version used by the compiler (<a href="https://github.com/rust-lang/rust/pull/139934">#139934</a>)</a></h3> |
| <p>For <code>compiler-builtins</code> this meant a version bump, in other cases it may be a git submodule update.</p> |
| <h3 id="step-4-remove-the-old-behavior-from-the-compiler-139753"><a class="header" href="#step-4-remove-the-old-behavior-from-the-compiler-139753">Step 4: remove the old behavior from the compiler (<a href="https://github.com/rust-lang/rust/pull/139753">#139753</a>)</a></h3> |
| <p>The updated crate can now be used. In this example that meant that the old behavior could be removed.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="high-level-compiler-architecture"><a class="header" href="#high-level-compiler-architecture">High-Level Compiler Architecture</a></h1> |
| <p>The remaining parts of this guide discuss how the compiler works. They go |
| through everything from high-level structure of the compiler to how each stage |
| of compilation works. They should be friendly to both readers interested in the |
| end-to-end process of compilation <em>and</em> readers interested in learning about a |
| specific system they wish to contribute to. If anything is unclear, feel free |
| to file an issue on the <a href="https://github.com/rust-lang/rustc-dev-guide/issues">rustc-dev-guide |
| repo</a> or contact the compiler |
| team, as detailed in <a href="./compiler-team.html">this chapter from Part 1</a>.</p> |
| <p>In this part, we will look at the high-level architecture of the compiler. In |
| particular, we will look at three overarching design choices that impact the |
| whole compiler: the query system, incremental compilation, and interning.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="overview-of-the-compiler"><a class="header" href="#overview-of-the-compiler">Overview of the compiler</a></h1> |
| <p>This chapter is about the overall process of compiling a program -- how |
| everything fits together.</p> |
| <p>The Rust compiler is special in two ways: it does things to your code that |
| other compilers don't do (e.g. borrow-checking) and it has a lot of |
| unconventional implementation choices (e.g. queries). We will talk about these |
| in turn in this chapter, and in the rest of the guide, we will look at the |
| individual pieces in more detail.</p> |
| <h2 id="what-the-compiler-does-to-your-code"><a class="header" href="#what-the-compiler-does-to-your-code">What the compiler does to your code</a></h2> |
| <p>So first, let's look at what the compiler does to your code. For now, we will |
| avoid mentioning how the compiler implements these steps except as needed.</p> |
| <h3 id="invocation"><a class="header" href="#invocation">Invocation</a></h3> |
| <p>Compilation begins when a user writes a Rust source program in text and invokes |
| the <code>rustc</code> compiler on it. The work that the compiler needs to perform is |
| defined by command-line options. For example, it is possible to enable nightly |
| features (<code>-Z</code> flags), perform <code>check</code>-only builds, or emit the LLVM |
| Intermediate Representation (<code>LLVM-IR</code>) rather than executable machine code. |
| The <code>rustc</code> executable call may be indirect through the use of <code>cargo</code>.</p> |
| <p>Command line argument parsing occurs in the <a href="rustc-driver/intro.html"><code>rustc_driver</code></a>. This crate |
| defines the compile configuration that is requested by the user and passes it |
| to the rest of the compilation process as a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Config.html"><code>rustc_interface::Config</code></a>.</p> |
| <h3 id="lexing-and-parsing"><a class="header" href="#lexing-and-parsing">Lexing and parsing</a></h3> |
| <p>The raw Rust source text is analyzed by a low-level <em>lexer</em> located in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a>. At this stage, the source text is turned into a stream of |
| atomic source code units known as <em>tokens</em>. The <code>lexer</code> supports the |
| Unicode character encoding.</p> |
| <p>The token stream passes through a higher-level lexer located in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html"><code>rustc_parse</code></a> to prepare for the next stage of the compile process. The |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html"><code>Lexer</code></a> <code>struct</code> is used at this stage to perform a set of validations |
| and turn strings into interned symbols (<em>interning</em> is discussed later). |
| <a href="https://en.wikipedia.org/wiki/String_interning">String interning</a> is a way of storing only one immutable |
| copy of each distinct string value.</p> |
| <p>The lexer has a small interface and doesn't depend directly on the diagnostic |
| infrastructure in <code>rustc</code>. Instead it provides diagnostics as plain data which |
| are emitted in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/index.html"><code>rustc_parse::lexer</code></a> as real diagnostics. The <code>lexer</code> |
| preserves full fidelity information for both IDEs and procedural macros |
| (sometimes referred to as "proc-macros").</p> |
| <p>The <em>parser</em> <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html">translates the token stream from the <code>lexer</code> into an Abstract Syntax |
| Tree (AST)</a>. It uses a recursive descent (top-down) approach to syntax |
| analysis. The crate entry points for the <code>parser</code> are the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.parse_crate_mod"><code>Parser::parse_crate_mod()</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.parse_mod"><code>Parser::parse_mod()</code></a> |
| methods found in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html"><code>rustc_parse::parser::Parser</code></a>. The external module parsing |
| entry point is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/module/fn.parse_external_mod.html"><code>rustc_expand::module::parse_external_mod</code></a>. |
| And the macro-<code>parser</code> entry point is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.parse_nonterminal"><code>Parser::parse_nonterminal()</code></a>.</p> |
| <p>Parsing is performed with a set of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html"><code>parser</code></a> utility methods including <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.bump"><code>bump</code></a>, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.check"><code>check</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.eat"><code>eat</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.expect"><code>expect</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.look_ahead"><code>look_ahead</code></a>.</p> |
| <p>Parsing is organized by semantic construct. Separate |
| <code>parse_*</code> methods can be found in the <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_parse/src/parser"><code>rustc_parse</code></a> |
| directory. The source file name follows the construct name. For example, the |
| following files are found in the <code>parser</code>:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_parse/src/parser/expr.rs"><code>expr.rs</code></a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_parse/src/parser/pat.rs"><code>pat.rs</code></a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_parse/src/parser/ty.rs"><code>ty.rs</code></a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_parse/src/parser/stmt.rs"><code>stmt.rs</code></a></li> |
| </ul> |
| <p>This naming scheme is used across many compiler stages. You will find either a |
| file or directory with the same name across the parsing, lowering, type |
| checking, <a href="./thir.html">Typed High-level Intermediate Representation (<code>THIR</code>)</a> lowering, and |
| <a href="mir/index.html">Mid-level Intermediate Representation (<code>MIR</code>)</a> building sources.</p> |
| <p>Macro-expansion, <code>AST</code>-validation, name-resolution, and early linting also take |
| place during the lexing and parsing stage.</p> |
| <p>The <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_ast/index.html"><code>rustc_ast::ast</code></a>::{<a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_ast/ast/struct.Crate.html"><code>Crate</code></a>, <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_ast/ast/struct.Expr.html"><code>Expr</code></a>, <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_ast/ast/struct.Pat.html"><code>Pat</code></a>, ...} <code>AST</code> nodes are |
| returned from the parser while the standard <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html"><code>Diag</code></a> API is used |
| for error handling. Generally Rust's compiler will try to recover from errors |
| by parsing a superset of Rust's grammar, while also emitting an error type.</p> |
| <h3 id="ast-lowering"><a class="header" href="#ast-lowering"><code>AST</code> lowering</a></h3> |
| <p>Next the <code>AST</code> is converted into <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/index.html">High-Level Intermediate Representation |
| (<code>HIR</code>)</a>, a more compiler-friendly representation of the <code>AST</code>. This process |
| is called "lowering" and involves a lot of desugaring (the expansion and |
| formalizing of shortened or abbreviated syntax constructs) of things like loops |
| and <code>async fn</code>.</p> |
| <p>We then use the <code>HIR</code> to do <a href="type-inference.html"><em>type inference</em></a> (the process of automatic |
| detection of the type of an expression), <a href="traits/resolution.html"><em>trait solving</em></a> (the process of |
| pairing up an impl with each reference to a <code>trait</code>), and <a href="type-checking.html"><em>type checking</em></a>. Type |
| checking is the process of converting the types found in the <code>HIR</code> (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Ty.html"><code>hir::Ty</code></a>), |
| which represent what the user wrote, into the internal representation used by |
| the compiler (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>Ty<'tcx></code></a>). It's called type checking because the information |
| is used to verify the type safety, correctness and coherence of the types used |
| in the program.</p> |
| <h3 id="mir-lowering"><a class="header" href="#mir-lowering"><code>MIR</code> lowering</a></h3> |
| <p>The <code>HIR</code> is further lowered to <code>MIR</code> |
| (used for <a href="borrow_check.html">borrow checking</a>) by constructing the <code>THIR</code> (an even more desugared <code>HIR</code> used for |
| pattern and exhaustiveness checking) to convert into <code>MIR</code>.</p> |
| <p>We do <a href="mir/optimizations.html">many optimizations on the MIR</a> because it is generic and that |
| improves later code generation and compilation speed. It is easier to do some |
| optimizations at <code>MIR</code> level than at <code>LLVM-IR</code> level. For example LLVM doesn't seem |
| to be able to optimize the pattern the <a href="https://github.com/rust-lang/rust/pull/66282"><code>simplify_try</code></a> <code>MIR</code>-opt looks for.</p> |
| <p>Rust code is also <a href="https://en.wikipedia.org/wiki/Monomorphization"><em>monomorphized</em></a> during code generation, which means making |
| copies of all the generic code with the type parameters replaced by concrete |
| types. To do this, we need to collect a list of what concrete types to generate |
| code for. This is called <em>monomorphization collection</em> and it happens at the |
| <code>MIR</code> level.</p> |
| <h3 id="code-generation"><a class="header" href="#code-generation">Code generation</a></h3> |
| <p>We then begin what is simply called <em>code generation</em> or <em>codegen</em>. The <a href="backend/codegen.html">code |
| generation stage</a> is when higher-level representations of source are |
| turned into an executable binary. Since <code>rustc</code> uses LLVM for code generation, |
| the first step is to convert the <code>MIR</code> to <code>LLVM-IR</code>. This is where the <code>MIR</code> is |
| actually monomorphized. The <code>LLVM-IR</code> is passed to LLVM, which does a lot more |
| optimizations on it, emitting machine code which is basically assembly code |
| with additional low-level types and annotations added (e.g. an ELF object or |
| <code>WASM</code>). The different libraries/binaries are then linked together to produce |
| the final binary.</p> |
| <h2 id="how-it-does-it"><a class="header" href="#how-it-does-it">How it does it</a></h2> |
| <p>Now that we have a high-level view of what the compiler does to your code, |
| let's take a high-level view of <em>how</em> it does all that stuff. There are a lot |
| of constraints and conflicting goals that the compiler needs to |
| satisfy/optimize for. For example,</p> |
| <ul> |
| <li>Compilation speed: how fast is it to compile a program? More/better |
| compile-time analyses often means compilation is slower. |
| <ul> |
| <li>Also, we want to support incremental compilation, so we need to take that |
| into account. How can we keep track of what work needs to be redone and |
| what can be reused if the user modifies their program? |
| <ul> |
| <li>Also we can't store too much stuff in the incremental cache because |
| it would take a long time to load from disk and it could take a lot |
| of space on the user's system...</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>Compiler memory usage: while compiling a program, we don't want to use more |
| memory than we need.</li> |
| <li>Program speed: how fast is your compiled program? More/better compile-time |
| analyses often means the compiler can do better optimizations.</li> |
| <li>Program size: how large is the compiled binary? Similar to the previous |
| point.</li> |
| <li>Compiler compilation speed: how long does it take to compile the compiler? |
| This impacts contributors and compiler maintenance.</li> |
| <li>Implementation complexity: building a compiler is one of the hardest |
| things a person/group can do, and Rust is not a very simple language, so how |
| do we make the compiler's code base manageable?</li> |
| <li>Compiler correctness: the binaries produced by the compiler should do what |
| the input programs says they do, and should continue to do so despite the |
| tremendous amount of change constantly going on.</li> |
| <li>Integration: a number of other tools need to use the compiler in |
| various ways (e.g. <code>cargo</code>, <code>clippy</code>, <code>MIRI</code>) that must be supported.</li> |
| <li>Compiler stability: the compiler should not crash or fail ungracefully on the |
| stable channel.</li> |
| <li>Rust stability: the compiler must respect Rust's stability guarantees by not |
| breaking programs that previously compiled despite the many changes that are |
| always going on to its implementation.</li> |
| <li>Limitations of other tools: <code>rustc</code> uses LLVM in its backend, and LLVM has some |
| strengths we leverage and some aspects we need to work around.</li> |
| </ul> |
| <p>So, as you continue your journey through the rest of the guide, keep these |
| things in mind. They will often inform decisions that we make.</p> |
| <h3 id="intermediate-representations"><a class="header" href="#intermediate-representations">Intermediate representations</a></h3> |
| <p>As with most compilers, <code>rustc</code> uses some intermediate representations (IRs) to |
| facilitate computations. In general, working directly with the source code is |
| extremely inconvenient and error-prone. Source code is designed to be human-friendly while at |
| the same time being unambiguous, but it's less convenient for doing something |
| like, say, type checking.</p> |
| <p>Instead most compilers, including <code>rustc</code>, build some sort of IR out of the |
| source code which is easier to analyze. <code>rustc</code> has a few IRs, each optimized |
| for different purposes:</p> |
| <ul> |
| <li>Token stream: the lexer produces a stream of tokens directly from the source |
| code. This stream of tokens is easier for the parser to deal with than raw |
| text.</li> |
| <li>Abstract Syntax Tree (<code>AST</code>): the abstract syntax tree is built from the stream |
| of tokens produced by the lexer. It represents |
| pretty much exactly what the user wrote. It helps to do some syntactic sanity |
| checking (e.g. checking that a type is expected where the user wrote one).</li> |
| <li>High-level IR (HIR): This is a sort of desugared <code>AST</code>. It's still close |
| to what the user wrote syntactically, but it includes some implicit things |
| such as some elided lifetimes, etc. This IR is amenable to type checking.</li> |
| <li>Typed <code>HIR</code> (THIR) <em>formerly High-level Abstract IR (HAIR)</em>: This is an |
| intermediate between <code>HIR</code> and MIR. It is like the <code>HIR</code> but it is fully typed |
| and a bit more desugared (e.g. method calls and implicit dereferences are |
| made fully explicit). As a result, it is easier to lower to <code>MIR</code> from <code>THIR</code> than |
| from HIR.</li> |
| <li>Middle-level IR (<code>MIR</code>): This IR is basically a Control-Flow Graph (CFG). A CFG |
| is a type of diagram that shows the basic blocks of a program and how control |
| flow can go between them. Likewise, <code>MIR</code> also has a bunch of basic blocks with |
| simple typed statements inside them (e.g. assignment, simple computations, |
| etc) and control flow edges to other basic blocks (e.g., calls, dropping |
| values). <code>MIR</code> is used for borrow checking and other |
| important dataflow-based checks, such as checking for uninitialized values. |
| It is also used for a series of optimizations and for constant evaluation (via |
| <code>MIRI</code>). Because <code>MIR</code> is still generic, we can do a lot of analyses here more |
| efficiently than after monomorphization.</li> |
| <li><code>LLVM-IR</code>: This is the standard form of all input to the LLVM compiler. <code>LLVM-IR</code> |
| is a sort of typed assembly language with lots of annotations. It's |
| a standard format that is used by all compilers that use LLVM (e.g. the clang |
| C compiler also outputs <code>LLVM-IR</code>). <code>LLVM-IR</code> is designed to be easy for other |
| compilers to emit and also rich enough for LLVM to run a bunch of |
| optimizations on it.</li> |
| </ul> |
| <p>One other thing to note is that many values in the compiler are <em>interned</em>. |
| This is a performance and memory optimization in which we allocate the values in |
| a special allocator called an |
| <em><a href="https://en.wikipedia.org/wiki/Region-based_memory_management">arena</a></em>. Then, we pass |
| around references to the values allocated in the arena. This allows us to make |
| sure that identical values (e.g. types in your program) are only allocated once |
| and can be compared cheaply by comparing pointers. Many of the intermediate |
| representations are interned.</p> |
| <h3 id="queries"><a class="header" href="#queries">Queries</a></h3> |
| <p>The first big implementation choice is Rust's use of the <em>query</em> system in its |
| compiler. The Rust compiler <em>is not</em> organized as a series of passes over the |
| code which execute sequentially. The Rust compiler does this to make |
| incremental compilation possible -- that is, if the user makes a change to |
| their program and recompiles, we want to do as little redundant work as |
| possible to output the new binary.</p> |
| <p>In <code>rustc</code>, all the major steps above are organized as a bunch of queries that |
| call each other. For example, there is a query to ask for the type of something |
| and another to ask for the optimized <code>MIR</code> of a function. These queries can call |
| each other and are all tracked through the query system. The results of the |
| queries are cached on disk so that the compiler can tell which queries' results |
| changed from the last compilation and only redo those. This is how incremental |
| compilation works.</p> |
| <p>In principle, for the query-fied steps, we do each of the above for each item |
| individually. For example, we will take the <code>HIR</code> for a function and use queries |
| to ask for the <code>LLVM-IR</code> for that HIR. This drives the generation of optimized |
| <code>MIR</code>, which drives the borrow checker, which drives the generation of <code>MIR</code>, and |
| so on.</p> |
| <p>... except that this is very over-simplified. In fact, some queries are not |
| cached on disk, and some parts of the compiler have to run for all code anyway |
| for correctness even if the code is dead code (e.g. the borrow checker). For |
| example, <a href="https://github.com/rust-lang/rust/blob/e69c7306e2be08939d95f14229e3f96566fb206c/compiler/rustc_interface/src/passes.rs#L791">currently the <code>mir_borrowck</code> query is first executed on all functions |
| of a crate.</a> Then the codegen backend invokes the |
| <code>collect_and_partition_mono_items</code> query, which first recursively requests the |
| <code>optimized_mir</code> for all reachable functions, which in turn runs <code>mir_borrowck</code> |
| for that function and then creates codegen units. This kind of split will need |
| to remain to ensure that unreachable functions still have their errors emitted.</p> |
| <p>Moreover, the compiler wasn't originally built to use a query system; the query |
| system has been retrofitted into the compiler, so parts of it are not query-fied |
| yet. Also, LLVM isn't our code, so that isn't querified either. The plan is to |
| eventually query-fy all of the steps listed in the previous section, |
| but as of <!-- date-check --> November 2022, only the steps between <code>HIR</code> and |
| <code>LLVM-IR</code> are query-fied. That is, lexing, parsing, name resolution, and macro |
| expansion are done all at once for the whole program.</p> |
| <p>One other thing to mention here is the all-important "typing context", |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a>, which is a giant struct that is at the center of all things. |
| (Note that the name is mostly historic. This is <em>not</em> a "typing context" in the |
| sense of <code>Γ</code> or <code>Δ</code> from type theory. The name is retained because that's what |
| the name of the struct is in the source code.) All |
| queries are defined as methods on the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a> type, and the in-memory query |
| cache is stored there too. In the code, there is usually a variable called |
| <code>tcx</code> which is a handle on the typing context. You will also see lifetimes with |
| the name <code>'tcx</code>, which means that something is tied to the lifetime of the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a> (usually it is stored or interned there).</p> |
| <p>For more information about queries in the compiler, see <a href="./query.html">the queries chapter</a>.</p> |
| <h3 id="tyty"><a class="header" href="#tyty"><code>ty::Ty</code></a></h3> |
| <p>Types are really important in Rust, and they form the core of a lot of compiler |
| analyses. The main type (in the compiler) that represents types (in the user's |
| program) is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>rustc_middle::ty::Ty</code></a>. This is so important that we have a whole chapter |
| on <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>ty::Ty</code></a>, but for now, we just want to mention that it exists and is the way |
| <code>rustc</code> represents types!</p> |
| <p>Also note that the <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_middle/ty/index.html"><code>rustc_middle::ty</code></a> module defines the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a> struct we mentioned before.</p> |
| <h3 id="parallelism"><a class="header" href="#parallelism">Parallelism</a></h3> |
| <p>Compiler performance is a problem that we would like to improve on |
| (and are always working on). One aspect of that is parallelizing |
| <code>rustc</code> itself.</p> |
| <p>Currently, there is only one part of rustc that is parallel by default: |
| <a href="./parallel-rustc.html#Codegen">code generation</a>.</p> |
| <p>However, the rest of the compiler is still not yet parallel. There have been |
| lots of efforts spent on this, but it is generally a hard problem. The current |
| approach is to turn <a href="https://doc.rust-lang.org/std/cell/struct.RefCell.html"><code>RefCell</code></a>s into <a href="https://doc.rust-lang.org/std/sync/struct.Mutex.html"><code>Mutex</code></a>s -- that is, we |
| switch to thread-safe internal mutability. However, there are ongoing |
| challenges with lock contention, maintaining query-system invariants under |
| concurrency, and the complexity of the code base. One can try out the current |
| work by enabling parallel compilation in <code>bootstrap.toml</code>. It's still early days, |
| but there are already some promising performance improvements.</p> |
| <h3 id="bootstrapping"><a class="header" href="#bootstrapping">Bootstrapping</a></h3> |
| <p><code>rustc</code> itself is written in Rust. So how do we compile the compiler? We use an |
| older compiler to compile the newer compiler. This is called <a href="https://en.wikipedia.org/wiki/Bootstrapping_(compilers)"><em>bootstrapping</em></a>.</p> |
| <p>Bootstrapping has a lot of interesting implications. For example, it means |
| that one of the major users of Rust is the Rust compiler, so we are |
| constantly testing our own software ("eating our own dogfood").</p> |
| <p>For more details on bootstrapping, see |
| <a href="building/bootstrapping/intro.html">the bootstrapping section of the guide</a>.</p> |
| <!-- |
| # Unresolved Questions |
| |
| - Does LLVM ever do optimizations in debug builds? |
| - How do I explore phases of the compile process in my own sources (lexer, |
| parser, HIR, etc)? - e.g., `cargo rustc -- -Z unpretty=hir-tree` allows you to |
| view `HIR` representation |
| - What is the main source entry point for `X`? |
| - Where do phases diverge for cross-compilation to machine code across |
| different platforms? |
| --> |
| <h1 id="references"><a class="header" href="#references">References</a></h1> |
| <ul> |
| <li>Command line parsing |
| <ul> |
| <li>Guide: <a href="rustc-driver/intro.html">The Rustc Driver and Interface</a></li> |
| <li>Driver definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/"><code>rustc_driver</code></a></li> |
| <li>Main entry point: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/config/fn.build_session_options.html"><code>rustc_session::config::build_session_options</code></a></li> |
| </ul> |
| </li> |
| <li>Lexical Analysis: Lex the user program to a stream of tokens |
| <ul> |
| <li>Guide: <a href="the-parser.html">Lexing and Parsing</a></li> |
| <li>Lexer definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a></li> |
| <li>Main entry point: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/cursor/struct.Cursor.html#method.advance_token"><code>rustc_lexer::cursor::Cursor::advance_token</code></a></li> |
| </ul> |
| </li> |
| <li>Parsing: Parse the stream of tokens to an Abstract Syntax Tree (AST) |
| <ul> |
| <li>Guide: <a href="the-parser.html">Lexing and Parsing</a></li> |
| <li>Guide: <a href="macro-expansion.html">Macro Expansion</a></li> |
| <li>Guide: <a href="name-resolution.html">Name Resolution</a></li> |
| <li>Parser definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html"><code>rustc_parse</code></a></li> |
| <li>Main entry points: |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/passes/fn.parse.html">Entry point for first file in crate</a></li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/module/fn.parse_external_mod.html">Entry point for outline module parsing</a></li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html#method.parse_nonterminal">Entry point for macro fragments</a></li> |
| </ul> |
| </li> |
| <li><code>AST</code> definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ast/index.html"><code>rustc_ast</code></a></li> |
| <li>Feature gating: <strong>TODO</strong></li> |
| <li>Early linting: <strong>TODO</strong></li> |
| </ul> |
| </li> |
| <li>The High Level Intermediate Representation (HIR) |
| <ul> |
| <li>Guide: <a href="hir.html">The HIR</a></li> |
| <li>Guide: <a href="hir.html#identifiers-in-the-hir">Identifiers in the HIR</a></li> |
| <li>Guide: <a href="hir.html#the-hir-map">The <code>HIR</code> Map</a></li> |
| <li>Guide: <a href="./hir/lowering.html">Lowering <code>AST</code> to <code>HIR</code></a></li> |
| <li>How to view <code>HIR</code> representation for your code <code>cargo rustc -- -Z unpretty=hir-tree</code></li> |
| <li>Rustc <code>HIR</code> definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/index.html"><code>rustc_hir</code></a></li> |
| <li>Main entry point: <strong>TODO</strong></li> |
| <li>Late linting: <strong>TODO</strong></li> |
| </ul> |
| </li> |
| <li>Type Inference |
| <ul> |
| <li>Guide: <a href="type-inference.html">Type Inference</a></li> |
| <li>Guide: <a href="ty.html">The ty Module: Representing Types</a> (semantics)</li> |
| <li>Main entry point (type inference): <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/struct.InferCtxtBuilder.html#method.enter"><code>InferCtxtBuilder::enter</code></a></li> |
| <li>Main entry point (type checking bodies): <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.typeck">the <code>typeck</code> query</a> |
| <ul> |
| <li>These two functions can't be decoupled.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>The Mid Level Intermediate Representation (MIR) |
| <ul> |
| <li>Guide: <a href="mir/index.html">The <code>MIR</code> (Mid level IR)</a></li> |
| <li>Definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/index.html"><code>rustc_middle/src/mir</code></a></li> |
| <li>Definition of sources that manipulates the MIR: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/index.html"><code>rustc_mir_build</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/index.html"><code>rustc_mir_dataflow</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/index.html"><code>rustc_mir_transform</code></a></li> |
| </ul> |
| </li> |
| <li>The Borrow Checker |
| <ul> |
| <li>Guide: <a href="borrow_check.html">MIR Borrow Check</a></li> |
| <li>Definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/index.html"><code>rustc_borrowck</code></a></li> |
| <li>Main entry point: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/fn.mir_borrowck.html"><code>mir_borrowck</code> query</a></li> |
| </ul> |
| </li> |
| <li><code>MIR</code> Optimizations |
| <ul> |
| <li>Guide: <a href="mir/optimizations.html">MIR Optimizations</a></li> |
| <li>Definition: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/index.html"><code>rustc_mir_transform</code></a></li> |
| <li>Main entry point: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/fn.optimized_mir.html"><code>optimized_mir</code> query</a></li> |
| </ul> |
| </li> |
| <li>Code Generation |
| <ul> |
| <li>Guide: <a href="backend/codegen.html">Code Generation</a></li> |
| <li>Generating Machine Code from <code>LLVM-IR</code> with LLVM - <strong>TODO: reference?</strong></li> |
| <li>Main entry point: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/fn.codegen_crate.html"><code>rustc_codegen_ssa::base::codegen_crate</code></a> |
| <ul> |
| <li>This monomorphizes and produces <code>LLVM-IR</code> for one codegen unit. It then |
| starts a background thread to run LLVM, which must be joined later.</li> |
| <li>Monomorphization happens lazily via <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#method.monomorphize"><code>FunctionCx::monomorphize</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/fn.codegen_instance.html"><code>rustc_codegen_ssa::base::codegen_instance </code></a></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="high-level-overview-of-the-compiler-source"><a class="header" href="#high-level-overview-of-the-compiler-source">High-level overview of the compiler source</a></h1> |
| <p>Now that we have <a href="./overview.html">seen what the compiler does</a>, |
| let's take a look at the structure of the <a href="https://github.com/rust-lang/rust"><code>rust-lang/rust</code></a> repository, |
| where the rustc source code lives.</p> |
| <blockquote> |
| <p>You may find it helpful to read the <a href="./overview.html">"Overview of the compiler"</a> |
| chapter, which introduces how the compiler works, before this one.</p> |
| </blockquote> |
| <h2 id="workspace-structure"><a class="header" href="#workspace-structure">Workspace structure</a></h2> |
| <p>The <a href="https://github.com/rust-lang/rust"><code>rust-lang/rust</code></a> repository consists of a single large cargo workspace |
| containing the compiler, the standard libraries (<a href="https://github.com/rust-lang/rust/tree/HEAD/library/core"><code>core</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/library/alloc"><code>alloc</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/library/std"><code>std</code></a>, |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/library/proc_macro"><code>proc_macro</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/src/etc"><code>etc</code></a>), and <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a>, along with the build system and a |
| bunch of tools and submodules for building a full Rust distribution.</p> |
| <p>The repository consists of three main directories:</p> |
| <ul> |
| <li> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/compiler"><code>compiler/</code></a> contains the source code for <code>rustc</code>. It consists of many crates |
| that together make up the compiler.</p> |
| </li> |
| <li> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/library"><code>library/</code></a> contains the standard libraries (<a href="https://github.com/rust-lang/rust/tree/HEAD/library/core"><code>core</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/library/alloc"><code>alloc</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/library/std"><code>std</code></a>, |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/library/proc_macro"><code>proc_macro</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/library/test"><code>test</code></a>), as well as the Rust runtime (<a href="https://github.com/rust-lang/backtrace-rs/"><code>backtrace</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/library/rtstartup"><code>rtstartup</code></a>, |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/library/std/src/rt.rs"><code>lang_start</code></a>).</p> |
| </li> |
| <li> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/tests"><code>tests/</code></a> contains the compiler tests.</p> |
| </li> |
| <li> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/src"><code>src/</code></a> contains the source code for <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a>, <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/clippy"><code>clippy</code></a>, <a href="https://github.com/rust-lang/cargo"><code>cargo</code></a>, the build system, |
| language docs, etc.</p> |
| </li> |
| </ul> |
| <h2 id="compiler"><a class="header" href="#compiler">Compiler</a></h2> |
| <p>The compiler is implemented in the various <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler"><code>compiler/</code></a> crates. |
| The <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler"><code>compiler/</code></a> crates all have names starting with <code>rustc_*</code>. These are a |
| collection of around 50 interdependent crates ranging in size from tiny to |
| huge. There is also the <code>rustc</code> crate which is the actual binary (i.e. the |
| <code>main</code> function); it doesn't actually do anything besides calling the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/index.html"><code>rustc_driver</code></a> crate, which drives the various parts of compilation in other |
| crates.</p> |
| <p>The dependency order of these crates is complex, but roughly it is |
| something like this:</p> |
| <ol> |
| <li><code>rustc</code> (the binary) calls <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/fn.main.html"><code>rustc_driver::main</code></a>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/index.html"><code>rustc_driver</code></a> depends on a lot of other crates, but the main one is |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> depends on most of the other compiler crates. It is a |
| fairly generic interface for driving the whole compilation.</li> |
| <li>Most of the other <code>rustc_*</code> crates depend on <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/index.html"><code>rustc_middle</code></a>, which defines |
| a lot of central data structures in the compiler.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/index.html"><code>rustc_middle</code></a> and most of the other crates depend on a handful of crates |
| representing the early parts of the compiler (e.g. the parser), fundamental |
| data structures (e.g. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html"><code>Span</code></a>), or error reporting: |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/index.html"><code>rustc_data_structures</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/index.html"><code>rustc_span</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/index.html"><code>rustc_errors</code></a>, etc.</li> |
| </ol> |
| <p>You can see the exact dependencies by running <code>cargo tree</code>, |
| just like you would for any other Rust package:</p> |
| <pre><code class="language-console">cargo tree --package rustc_driver |
| </code></pre> |
| <p>One final thing: <a href="https://github.com/rust-lang/rust/tree/HEAD/src/"><code>src/llvm-project</code></a> is a submodule for our fork of LLVM. |
| During bootstrapping, LLVM is built and the <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_llvm"><code>compiler/rustc_llvm</code></a> crate |
| contains Rust wrappers around LLVM (which is written in C++), so that the |
| compiler can interface with it.</p> |
| <p>Most of this book is about the compiler, so we won't have any further |
| explanation of these crates here.</p> |
| <h3 id="big-picture"><a class="header" href="#big-picture">Big picture</a></h3> |
| <p>The dependency structure of the compiler is influenced by two main factors:</p> |
| <ol> |
| <li>Organization. The compiler is a <em>huge</em> codebase; it would be an impossibly |
| large crate. In part, the dependency structure reflects the code structure |
| of the compiler.</li> |
| <li>Compile-time. By breaking the compiler into multiple crates, we can take |
| better advantage of incremental/parallel compilation using cargo. In |
| particular, we try to have as few dependencies between crates as possible so |
| that we don't have to rebuild as many crates if you change one.</li> |
| </ol> |
| <p>At the very bottom of the dependency tree are a handful of crates that are used |
| by the whole compiler (e.g. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/index.html"><code>rustc_span</code></a>). The very early parts of the |
| compilation process (e.g. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html">parsing and the Abstract Syntax Tree (<code>AST</code>)</a>) |
| depend on only these.</p> |
| <p>After the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html"><code>AST</code></a> is constructed and other early analysis is done, the |
| compiler's <a href="./query.html">query system</a> gets set up. The query system is set up in a |
| clever way using function pointers. This allows us to break dependencies |
| between crates, allowing more parallel compilation. The query system is defined |
| in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/index.html"><code>rustc_middle</code></a>, so nearly all subsequent parts of the compiler depend on |
| this crate. It is a really large crate, leading to long compile times. Some |
| efforts have been made to move stuff out of it with varying success. Another |
| side-effect is that sometimes related functionality gets scattered across |
| different crates. For example, linting functionality is found across earlier |
| parts of the crate, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/index.html"><code>rustc_lint</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/index.html"><code>rustc_middle</code></a>, and other places.</p> |
| <p>Ideally there would be fewer, more cohesive crates, with incremental and |
| parallel compilation making sure compile times stay reasonable. However, |
| incremental and parallel compilation haven't gotten good enough for that yet, |
| so breaking things into separate crates has been our solution so far.</p> |
| <p>At the top of the dependency tree is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/index.html"><code>rustc_driver</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> |
| which is an unstable wrapper around the query system helping drive various |
| stages of compilation. Other consumers of the compiler may use this interface |
| in different ways (e.g. <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a> or maybe eventually <code>rust-analyzer</code>). The |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/index.html"><code>rustc_driver</code></a> crate first parses command line arguments and then uses |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> to drive the compilation to completion.</p> |
| <h2 id="rustdoc-1"><a class="header" href="#rustdoc-1">rustdoc</a></h2> |
| <p>The bulk of <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a> is in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/index.html"><code>librustdoc</code></a>. However, the <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a> binary |
| itself is <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>src/tools/rustdoc</code></a>, which does nothing except call <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/fn.main.html"><code>rustdoc::main</code></a>.</p> |
| <p>There is also <code>JavaScript</code> and <code>CSS</code> for the docs in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc-js"><code>src/tools/rustdoc-js</code></a> |
| and <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc-themes"><code>src/tools/rustdoc-themes</code></a>. The type definitions for <code>--output-format=json</code> |
| are in a separate crate in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/rustdoc-json-types"><code>src/rustdoc-json-types</code></a>.</p> |
| <p>You can read more about <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a> in <a href="./rustdoc.html">this chapter</a>.</p> |
| <h2 id="tests-1"><a class="header" href="#tests-1">Tests</a></h2> |
| <p>The test suite for all of the above is in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests"><code>tests/</code></a>. You can read more |
| about the test suite <a href="./tests/intro.html">in this chapter</a>.</p> |
| <p>The test harness is in <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest"><code>src/tools/compiletest/</code></a>.</p> |
| <h2 id="build-system"><a class="header" href="#build-system">Build System</a></h2> |
| <p>There are a number of tools in the repository just for building the compiler, |
| standard library, <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a>, etc, along with testing, building a full Rust |
| distribution, etc.</p> |
| <p>One of the primary tools is <a href="https://github.com/rust-lang/rust/tree/HEAD/src/bootstrap"><code>src/bootstrap/</code></a>. You can read more about |
| bootstrapping <a href="./building/bootstrapping/intro.html">in this chapter</a>. The process may also use other tools |
| from <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools"><code>src/tools/</code></a>, such as <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/tidy"><code>tidy/</code></a> or <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/compiletest"><code>compiletest/</code></a>.</p> |
| <h2 id="standard-library"><a class="header" href="#standard-library">Standard library</a></h2> |
| <p>This code is fairly similar to most other Rust crates except that it must be |
| built in a special way because it can use unstable (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/"><code>nightly</code></a>) features. |
| The standard library is sometimes referred to as <a href="https://rust-lang.github.io/rfcs/0040-libstd-facade.html"><code>libstd or the "standard facade"</code></a>.</p> |
| <h2 id="other"><a class="header" href="#other">Other</a></h2> |
| <p>There are a lot of other things in the <code>rust-lang/rust</code> repo that are related |
| to building a full Rust distribution. Most of the time you don't need to worry about them.</p> |
| <p>These include:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/tree/HEAD/src/ci"><code>src/ci</code></a>: The CI configuration. This actually quite extensive because we |
| run a lot of tests on a lot of platforms.</li> |
| <li><a href="https://github.com/rust-lang/rust/tree/HEAD/src/doc"><code>src/doc</code></a>: Various documentation, including submodules for a few books.</li> |
| <li><a href="https://github.com/rust-lang/rust/tree/HEAD/src/etc"><code>src/etc</code></a>: Miscellaneous utilities.</li> |
| <li>And more...</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="queries-demand-driven-compilation"><a class="header" href="#queries-demand-driven-compilation">Queries: demand-driven compilation</a></h1> |
| <p>As described in <a href="overview.html#queries">Overview of the compiler</a>, the Rust compiler |
| is still (as of <!-- date-check --> July 2021) transitioning from a |
| traditional "pass-based" setup to a "demand-driven" system. The compiler query |
| system is the key to rustc's demand-driven organization. |
| The idea is pretty simple. Instead of entirely independent passes |
| (parsing, type-checking, etc.), a set of function-like <em>queries</em> |
| compute information about the input source. For example, |
| there is a query called <code>type_of</code> that, given the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/def_id/struct.DefId.html"><code>DefId</code></a> of |
| some item, will compute the type of that item and return it to you.</p> |
| <p>Query execution is <em>memoized</em>. The first time you invoke a |
| query, it will go do the computation, but the next time, the result is |
| returned from a hashtable. Moreover, query execution fits nicely into |
| <em>incremental computation</em>; the idea is roughly that, when you invoke a |
| query, the result <em>may</em> be returned to you by loading stored data |
| from disk.<sup class="footnote-reference" id="fr-incr-comp-detail-1"><a href="#footnote-incr-comp-detail">1</a></sup></p> |
| <p>Eventually, we want the entire compiler |
| control-flow to be query driven. There will effectively be one |
| top-level query (<code>compile</code>) that will run compilation on a crate; this |
| will in turn demand information about that crate, starting from the |
| <em>end</em>. For example:</p> |
| <ul> |
| <li>The <code>compile</code> query might demand to get a list of codegen-units |
| (i.e. modules that need to be compiled by LLVM).</li> |
| <li>But computing the list of codegen-units would invoke some subquery |
| that returns the list of all modules defined in the Rust source.</li> |
| <li>That query in turn would invoke something asking for the HIR.</li> |
| <li>This keeps going further and further back until we wind up doing the |
| actual parsing.</li> |
| </ul> |
| <p>Although this vision is not fully realized, large sections of the |
| compiler (for example, generating <a href="mir/index.html">MIR</a>) currently work exactly like this.</p> |
| <h2 id="invoking-queries"><a class="header" href="#invoking-queries">Invoking queries</a></h2> |
| <p>Invoking a query is simple. The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a> ("type context") struct offers a method |
| for each defined query. For example, to invoke the <code>type_of</code> |
| query, you would just do this:</p> |
| <pre><code class="language-rust ignore">let ty = tcx.type_of(some_def_id);</code></pre> |
| <h2 id="how-the-compiler-executes-a-query"><a class="header" href="#how-the-compiler-executes-a-query">How the compiler executes a query</a></h2> |
| <p>So you may be wondering what happens when you invoke a query |
| method. The answer is that, for each query, the compiler maintains a |
| cache – if your query has already been executed, then, the answer is |
| simple: we clone the return value out of the cache and return it |
| (therefore, you should try to ensure that the return types of queries |
| are cheaply cloneable; insert an <code>Rc</code> if necessary).</p> |
| <h3 id="providers"><a class="header" href="#providers">Providers</a></h3> |
| <p>If, however, the query is <em>not</em> in the cache, then the compiler will |
| call the corresponding <strong>provider</strong> function. A provider is a function |
| implemented in a specific module and <strong>manually registered</strong> into the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/struct.Providers.html"><code>Providers</code></a> struct during compiler initialization. |
| The macro system generates the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/struct.Providers.html"><code>Providers</code></a> struct, |
| which acts as a function table for all query implementations, where each |
| field is a function pointer to the actual provider.</p> |
| <p><strong>Note:</strong> The <code>Providers</code> struct is generated by macros and acts as a function table for all query implementations. |
| It is <strong>not</strong> a Rust trait, but a plain struct with function pointer fields.</p> |
| <p><strong>Providers are defined per-crate.</strong> The compiler maintains, |
| internally, a table of providers for every crate, at least |
| conceptually. Right now, there are really two sets: the providers for |
| queries about the <strong>local crate</strong> (that is, the one being compiled) |
| and providers for queries about <strong>external crates</strong> (that is, |
| dependencies of the local crate). Note that what determines the crate |
| that a query is targeting is not the <em>kind</em> of query, but the <em>key</em>. |
| For example, when you invoke <code>tcx.type_of(def_id)</code>, that could be a |
| local query or an external query, depending on what crate the <code>def_id</code> |
| is referring to (see the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/keys/trait.Key.html"><code>self::keys::Key</code></a> trait for more |
| information on how that works).</p> |
| <p>Providers always have the same signature:</p> |
| <pre><code class="language-rust ignore">fn provider<'tcx>( |
| tcx: TyCtxt<'tcx>, |
| key: QUERY_KEY, |
| ) -> QUERY_RESULT { |
| ... |
| }</code></pre> |
| <p>Providers take two arguments: the <code>tcx</code> and the query key. |
| They return the result of the query.</p> |
| <p>N.B. Most of the <code>rustc_*</code> crates only provide <strong>local |
| providers</strong>. Almost all <strong>extern providers</strong> wind up going through the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/index.html"><code>rustc_metadata</code> crate</a>, which loads the information |
| from the crate metadata. But in some cases there are crates that |
| provide queries for <em>both</em> local and external crates, in which case |
| they define both a <code>provide</code> and a <code>provide_extern</code> function, through |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/back/symbol_export/fn.wasm_import_module_map.html"><code>wasm_import_module_map</code></a>, that <code>rustc_driver</code> can invoke.</p> |
| <h3 id="how-providers-are-set-up"><a class="header" href="#how-providers-are-set-up">How providers are set up</a></h3> |
| <p>When the tcx is created, it is given the providers by its creator using |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/struct.Providers.html"><code>Providers</code></a> struct. This struct is generated by |
| the macros here, but it is basically a big list of function pointers:</p> |
| <pre><code class="language-rust ignore">struct Providers { |
| type_of: for<'tcx> fn(TyCtxt<'tcx>, DefId) -> Ty<'tcx>, |
| // ... one field for each query |
| }</code></pre> |
| <h4 id="how-are-providers-registered"><a class="header" href="#how-are-providers-registered">How are providers registered?</a></h4> |
| <p>The <code>Providers</code> struct is filled in during compiler initialization, mainly by the <code>rustc_driver</code> crate.<br /> |
| But the actual provider functions are implemented in various <code>rustc_*</code> crates (like <code>rustc_middle</code>, <code>rustc_hir_analysis</code>, etc).</p> |
| <p>To register providers, each crate exposes a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/fn.provide.html"><code>provide</code></a> function that looks like this:</p> |
| <pre><code class="language-rust ignore">pub fn provide(providers: &mut Providers) { |
| *providers = Providers { |
| type_of, |
| // ... add more providers here |
| ..*providers |
| }; |
| }</code></pre> |
| <ul> |
| <li>This function takes a mutable reference to the <code>Providers</code> struct and sets the fields to point to the correct provider functions.</li> |
| <li>You can also assign fields individually, e.g. <code>providers.type_of = type_of;</code>.</li> |
| </ul> |
| <h4 id="adding-a-new-provider"><a class="header" href="#adding-a-new-provider">Adding a new provider</a></h4> |
| <p>Suppose you want to add a new query called <code>fubar</code>. You would:</p> |
| <ol> |
| <li>Implement the provider function: |
| <pre><code class="language-rust ignore">fn fubar<'tcx>(tcx: TyCtxt<'tcx>, key: DefId) -> Fubar<'tcx> { ... }</code></pre> |
| </li> |
| <li>Register it in the <code>provide</code> function: |
| <pre><code class="language-rust ignore">pub fn provide(providers: &mut Providers) { |
| *providers = Providers { |
| fubar, |
| ..*providers |
| }; |
| }</code></pre> |
| </li> |
| </ol> |
| <hr /> |
| <h2 id="adding-a-new-query"><a class="header" href="#adding-a-new-query">Adding a new query</a></h2> |
| <p>How do you add a new query? |
| Defining a query takes place in two steps:</p> |
| <ol> |
| <li>Declare the query name, its arguments and description.</li> |
| <li>Supply query providers where needed.</li> |
| </ol> |
| <p>To declare the query name and arguments, you simply add an entry to |
| the big macro invocation in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/index.html"><code>compiler/rustc_middle/src/query/mod.rs</code></a>. |
| Then you need to add a documentation comment to it with some <em>internal</em> description. |
| Then, provide the <code>desc</code> attribute which contains a <em>user-facing</em> description of the query. |
| The <code>desc</code> attribute is shown to the user in query cycles.</p> |
| <p>This looks something like:</p> |
| <pre><code class="language-rust ignore">rustc_queries! { |
| /// Records the type of every item. |
| query type_of(key: DefId) -> Ty<'tcx> { |
| cache_on_disk_if { key.is_local() } |
| desc { |tcx| "computing the type of `{}`", tcx.def_path_str(key) } |
| } |
| ... |
| }</code></pre> |
| <p>A query definition has the following form:</p> |
| <pre><code class="language-rust ignore">query type_of(key: DefId) -> Ty<'tcx> { ... } |
| ^^^^^ ^^^^^^^ ^^^^^ ^^^^^^^^ ^^^ |
| | | | | | |
| | | | | query modifiers |
| | | | result type |
| | | query key type |
| | name of query |
| query keyword</code></pre> |
| <p>Let's go over these elements one by one:</p> |
| <ul> |
| <li><strong>Query keyword:</strong> indicates a start of a query definition.</li> |
| <li><strong>Name of query:</strong> the name of the query method |
| (<code>tcx.type_of(..)</code>). Also used as the name of a struct |
| (<code>ty::queries::type_of</code>) that will be generated to represent |
| this query.</li> |
| <li><strong>Query key type:</strong> the type of the argument to this query. |
| This type must implement the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/keys/trait.Key.html"><code>ty::query::keys::Key</code></a> trait, which |
| defines (for example) how to map it to a crate, and so forth.</li> |
| <li><strong>Result type of query:</strong> the type produced by this query. This type |
| should (a) not use <code>RefCell</code> or other interior mutability and (b) be |
| cheaply cloneable. Interning or using <code>Rc</code> or <code>Arc</code> is recommended for |
| non-trivial data types.<sup class="footnote-reference" id="fr-steal-1"><a href="#footnote-steal">2</a></sup></li> |
| <li><strong>Query modifiers:</strong> various flags and options that customize how the |
| query is processed (mostly with respect to <a href="queries/incremental-compilation-in-detail.html#query-modifiers">incremental compilation</a>).</li> |
| </ul> |
| <p>So, to add a query:</p> |
| <ul> |
| <li>Add an entry to <code>rustc_queries!</code> using the format above.</li> |
| <li>Link the provider by modifying the appropriate <code>provide</code> method; |
| or add a new one if needed and ensure that <code>rustc_driver</code> is invoking it.</li> |
| </ul> |
| <h2 id="external-links"><a class="header" href="#external-links">External links</a></h2> |
| <p>Related design ideas, and tracking issues:</p> |
| <ul> |
| <li>Design document: <a href="https://github.com/nikomatsakis/rustc-on-demand-incremental-design-doc/blob/master/0000-rustc-on-demand-and-incremental.md">On-demand Rustc incremental design doc</a></li> |
| <li>Tracking Issue: <a href="https://github.com/rust-lang/rust/issues/42293">"Red/Green" dependency tracking in compiler</a></li> |
| </ul> |
| <p>More discussion and issues:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/issues/42633">GitHub issue #42633</a></li> |
| <li><a href="https://internals.rust-lang.org/t/incremental-compilation-beta/4721">Incremental Compilation Beta</a></li> |
| <li><a href="https://blog.rust-lang.org/2016/09/08/incremental.html">Incremental Compilation Announcement</a></li> |
| </ul> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-incr-comp-detail"> |
| <p>The <a href="queries/incremental-compilation-in-detail.html">Incremental compilation in detail</a> chapter gives a more |
| in-depth description of what queries are and how they work. |
| If you intend to write a query of your own, this is a good read. <a href="#fr-incr-comp-detail-1">↩</a></p> |
| </li> |
| <li id="footnote-steal"> |
| <p>The one exception to those rules is the <code>ty::steal::Steal</code> type, |
| which is used to cheaply modify MIR in place. See the definition |
| of <code>Steal</code> for more details. New uses of <code>Steal</code> should <strong>not</strong> be |
| added without alerting <code>@rust-lang/compiler</code>. <a href="#fr-steal-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="the-query-evaluation-model-in-detail"><a class="header" href="#the-query-evaluation-model-in-detail">The Query Evaluation Model in detail</a></h1> |
| <p>This chapter provides a deeper dive into the abstract model queries are built on. |
| It does not go into implementation details but tries to explain |
| the underlying logic. The examples here, therefore, have been stripped down and |
| simplified and don't directly reflect the compilers internal APIs.</p> |
| <h2 id="what-is-a-query"><a class="header" href="#what-is-a-query">What is a query?</a></h2> |
| <p>Abstractly we view the compiler's knowledge about a given crate as a "database" |
| and queries are the way of asking the compiler questions about it, i.e. |
| we "query" the compiler's "database" for facts.</p> |
| <p>However, there's something special to this compiler database: It starts out empty |
| and is filled on-demand when queries are executed. Consequently, a query must |
| know how to compute its result if the database does not contain it yet. For |
| doing so, it can access other queries and certain input values that the database |
| is pre-filled with on creation.</p> |
| <p>A query thus consists of the following things:</p> |
| <ul> |
| <li>A name that identifies the query</li> |
| <li>A "key" that specifies what we want to look up</li> |
| <li>A result type that specifies what kind of result it yields</li> |
| <li>A "provider" which is a function that specifies how the result is to be |
| computed if it isn't already present in the database.</li> |
| </ul> |
| <p>As an example, the name of the <code>type_of</code> query is <code>type_of</code>, its query key is a |
| <code>DefId</code> identifying the item we want to know the type of, the result type is |
| <code>Ty<'tcx></code>, and the provider is a function that, given the query key and access |
| to the rest of the database, can compute the type of the item identified by the |
| key.</p> |
| <p>So in some sense a query is just a function that maps the query key to the |
| corresponding result. However, we have to apply some restrictions in order for |
| this to be sound:</p> |
| <ul> |
| <li>The key and result must be immutable values.</li> |
| <li>The provider function must be a pure function in the sense that for the same |
| key it must always yield the same result.</li> |
| <li>The only parameters a provider function takes are the key and a reference to |
| the "query context" (which provides access to the rest of the "database").</li> |
| </ul> |
| <p>The database is built up lazily by invoking queries. The query providers will |
| invoke other queries, for which the result is either already cached or computed |
| by calling another query provider. These query provider invocations |
| conceptually form a directed acyclic graph (DAG) at the leaves of which are |
| input values that are already known when the query context is created.</p> |
| <h2 id="cachingmemoization"><a class="header" href="#cachingmemoization">Caching/Memoization</a></h2> |
| <p>Results of query invocations are "memoized" which means that the query context |
| will cache the result in an internal table and, when the query is invoked with |
| the same query key again, will return the result from the cache instead of |
| running the provider again.</p> |
| <p>This caching is crucial for making the query engine efficient. Without |
| memoization the system would still be sound (that is, it would yield the same |
| results) but the same computations would be done over and over again.</p> |
| <p>Memoization is one of the main reasons why query providers have to be pure |
| functions. If calling a provider function could yield different results for |
| each invocation (because it accesses some global mutable state) then we could |
| not memoize the result.</p> |
| <h2 id="input-data"><a class="header" href="#input-data">Input data</a></h2> |
| <p>When the query context is created, it is still empty: No queries have been |
| executed, no results are cached. But the context already provides access to |
| "input" data, i.e. pieces of immutable data that were computed before the |
| context was created and that queries can access to do their computations.</p> |
| <p>As of <!-- date-check --> January 2021, this input data consists mainly of |
| the HIR map, upstream crate metadata, and the command-line options the compiler |
| was invoked with; but in the future inputs will just consist of command-line |
| options and a list of source files -- the HIR map will itself be provided by a |
| query which processes these source files.</p> |
| <p>Without inputs, queries would live in a void without anything to compute their |
| result from (remember, query providers only have access to other queries and |
| the context but not any other outside state or information).</p> |
| <p>For a query provider, input data and results of other queries look exactly the |
| same: It just tells the context "give me the value of X". Because input data |
| is immutable, the provider can rely on it being the same across |
| different query invocations, just as is the case for query results.</p> |
| <h2 id="an-example-execution-trace-of-some-queries"><a class="header" href="#an-example-execution-trace-of-some-queries">An example execution trace of some queries</a></h2> |
| <p>How does this DAG of query invocations come into existence? At some point |
| the compiler driver will create the, as yet empty, query context. It will then, |
| from outside of the query system, invoke the queries it needs to perform its |
| task. This looks something like the following:</p> |
| <pre><code class="language-rust ignore">fn compile_crate() { |
| let cli_options = ...; |
| let hir_map = ...; |
| |
| // Create the query context `tcx` |
| let tcx = TyCtxt::new(cli_options, hir_map); |
| |
| // Do type checking by invoking the type check query |
| tcx.type_check_crate(); |
| }</code></pre> |
| <p>The <code>type_check_crate</code> query provider would look something like the following:</p> |
| <pre><code class="language-rust ignore">fn type_check_crate_provider(tcx, _key: ()) { |
| let list_of_hir_items = tcx.hir_map.list_of_items(); |
| |
| for item_def_id in list_of_hir_items { |
| tcx.type_check_item(item_def_id); |
| } |
| }</code></pre> |
| <p>We see that the <code>type_check_crate</code> query accesses input data |
| (<code>tcx.hir_map.list_of_items()</code>) and invokes other queries |
| (<code>type_check_item</code>). The <code>type_check_item</code> |
| invocations will themselves access input data and/or invoke other queries, |
| so that in the end the DAG of query invocations will be built up backwards |
| from the node that was initially executed:</p> |
| <pre><code class="language-ignore"> (2) (1) |
| list_of_all_hir_items <----------------------------- type_check_crate() |
| | |
| (5) (4) (3) | |
| Hir(foo) <--- type_of(foo) <--- type_check_item(foo) <-------+ |
| | | |
| +-----------------+ | |
| | | |
| (7) v (6) (8) | |
| Hir(bar) <--- type_of(bar) <--- type_check_item(bar) <-------+ |
| |
| // (x) denotes invocation order |
| </code></pre> |
| <p>We also see that often a query result can be read from the cache: |
| <code>type_of(bar)</code> was computed for <code>type_check_item(foo)</code> so when |
| <code>type_check_item(bar)</code> needs it, it is already in the cache.</p> |
| <p>Query results stay cached in the query context as long as the context lives. |
| So if the compiler driver invoked another query later on, the above graph |
| would still exist and already executed queries would not have to be re-done.</p> |
| <h2 id="cycles"><a class="header" href="#cycles">Cycles</a></h2> |
| <p>Earlier we stated that query invocations form a DAG. However, it would be easy |
| to form a cyclic graph by, for example, having a query provider like the |
| following:</p> |
| <pre><code class="language-rust ignore">fn cyclic_query_provider(tcx, key) -> u32 { |
| // Invoke the same query with the same key again |
| tcx.cyclic_query(key) |
| }</code></pre> |
| <p>Since query providers are regular functions, this would behave much as expected: |
| Evaluation would get stuck in an infinite recursion. A query like this would not |
| be very useful either. However, sometimes certain kinds of invalid user input |
| can result in queries being called in a cyclic way. The query engine includes |
| a check for cyclic invocations of queries with the same input arguments. |
| And, because cycles are an irrecoverable error, will abort execution with a |
| "cycle error" message that tries to be human readable.</p> |
| <p>At some point the compiler had a notion of "cycle recovery", that is, one could |
| "try" to execute a query and if it ended up causing a cycle, proceed in some |
| other fashion. However, this was later removed because it is not entirely |
| clear what the theoretical consequences of this are, especially regarding |
| incremental compilation.</p> |
| <h2 id="steal-queries"><a class="header" href="#steal-queries">"Steal" Queries</a></h2> |
| <p>Some queries have their result wrapped in a <code>Steal<T></code> struct. These queries |
| behave exactly the same as regular with one exception: Their result is expected |
| to be "stolen" out of the cache at some point, meaning some other part of the |
| program is taking ownership of it and the result cannot be accessed anymore.</p> |
| <p>This stealing mechanism exists purely as a performance optimization because some |
| result values are too costly to clone (e.g. the MIR of a function). It seems |
| like result stealing would violate the condition that query results must be |
| immutable (after all we are moving the result value out of the cache) but it is |
| OK as long as the mutation is not observable. This is achieved by two things:</p> |
| <ul> |
| <li>Before a result is stolen, we make sure to eagerly run all queries that |
| might ever need to read that result. This has to be done manually by calling |
| those queries.</li> |
| <li>Whenever a query tries to access a stolen result, we make an ICE |
| (Internal Compiler Error) so that such a condition cannot go unnoticed.</li> |
| </ul> |
| <p>This is not an ideal setup because of the manual intervention needed, so it |
| should be used sparingly and only when it is well known which queries might |
| access a given result. In practice, however, stealing has not turned out to be |
| much of a maintenance burden.</p> |
| <p>To summarize: "Steal queries" break some of the rules in a controlled way. |
| There are checks in place that make sure that nothing can go silently wrong.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="incremental-compilation"><a class="header" href="#incremental-compilation">Incremental compilation</a></h1> |
| <p>The incremental compilation scheme is, in essence, a surprisingly |
| simple extension to the overall query system. We'll start by describing |
| a slightly simplified variant of the real thing – the "basic algorithm" – |
| and then describe some possible improvements.</p> |
| <h2 id="the-basic-algorithm"><a class="header" href="#the-basic-algorithm">The basic algorithm</a></h2> |
| <p>The basic algorithm is |
| called the <strong>red-green</strong> algorithm<sup class="footnote-reference" id="fr-salsa-1"><a href="#footnote-salsa">1</a></sup>. The high-level idea is |
| that, after each run of the compiler, we will save the results of all |
| the queries that we do, as well as the <strong>query DAG</strong>. The |
| <strong>query DAG</strong> is a <a href="https://en.wikipedia.org/wiki/Directed_acyclic_graph">DAG</a> that indexes which queries executed which |
| other queries. So, for example, there would be an <a href="https://en.wikipedia.org/wiki/Glossary_of_graph_theory_terms#edge">edge</a> from a query Q1 |
| to another query Q2 if computing Q1 required computing Q2 (note that |
| because queries cannot depend on themselves, this results in a DAG and |
| not a general graph).</p> |
| <blockquote> |
| <p><strong>NOTE</strong>: You might think of a query as simply the definition of a query. |
| A thing that you can invoke, a bit like a function, |
| and which either returns a cached result or actually executes the code.</p> |
| <p>If that's the way you think about queries, |
| it's good to know that in the following text, queries will be said to have colours. |
| Keep in mind though, that here the word query also refers to a certain invocation of |
| the query for a certain input. As you will read later, queries are fingerprinted based |
| on their arguments. The result of a query might change when we give it one argument |
| and be coloured red, while it stays the same for another argument and is thus green.</p> |
| <p>In short, the word query is here not just used to mean the definition of a query, |
| but also for a specific instance of that query with given arguments.</p> |
| </blockquote> |
| <p>On the next run of the compiler, then, we can sometimes reuse these |
| query results to avoid re-executing a query. We do this by assigning |
| every query a <strong>color</strong>:</p> |
| <ul> |
| <li>If a query is colored <strong>red</strong>, that means that its result during |
| this compilation has <strong>changed</strong> from the previous compilation.</li> |
| <li>If a query is colored <strong>green</strong>, that means that its result is |
| the <strong>same</strong> as the previous compilation.</li> |
| </ul> |
| <p>There are two key insights here:</p> |
| <ul> |
| <li>First, if all the inputs to query Q are colored green, then the |
| query Q <strong>must</strong> result in the same value as last time and hence |
| need not be re-executed (or else the compiler is not deterministic).</li> |
| <li>Second, even if some inputs to a query changes, it may be that it |
| <strong>still</strong> produces the same result as the previous compilation. In |
| particular, the query may only use part of its input. |
| <ul> |
| <li>Therefore, after executing a query, we always check whether it |
| produced the same result as the previous time. <strong>If it did,</strong> we |
| can still mark the query as green, and hence avoid re-executing |
| dependent queries.</li> |
| </ul> |
| </li> |
| </ul> |
| <h3 id="the-try-mark-green-algorithm"><a class="header" href="#the-try-mark-green-algorithm">The try-mark-green algorithm</a></h3> |
| <p>At the core of incremental compilation is an algorithm called |
| "try-mark-green". It has the job of determining the color of a given |
| query Q (which must not have yet been executed). In cases where Q has |
| red inputs, determining Q's color may involve re-executing Q so that |
| we can compare its output, but if all of Q's inputs are green, then we |
| can conclude that Q must be green without re-executing it or inspecting |
| its value at all. In the compiler, this allows us to avoid |
| deserializing the result from disk when we don't need it, and in fact |
| enables us to sometimes skip <em>serializing</em> the result as well |
| (see the refinements section below).</p> |
| <p>Try-mark-green works as follows:</p> |
| <ul> |
| <li>First check if the query Q was executed during the previous compilation. |
| <ul> |
| <li>If not, we can just re-execute the query as normal, and assign it the |
| color of red.</li> |
| </ul> |
| </li> |
| <li>If yes, then load the 'dependent queries' of Q.</li> |
| <li>If there is a saved result, then we load the <code>reads(Q)</code> vector from the |
| query DAG. The "reads" is the set of queries that Q executed during |
| its execution. |
| <ul> |
| <li>For each query R in <code>reads(Q)</code>, we recursively demand the color |
| of R using try-mark-green. |
| <ul> |
| <li>Note: it is important that we visit each node in <code>reads(Q)</code> in same order |
| as they occurred in the original compilation. See <a href="queries/incremental-compilation.html#dag">the section on the |
| query DAG below</a>.</li> |
| <li>If <strong>any</strong> of the nodes in <code>reads(Q)</code> wind up colored <strong>red</strong>, then Q is |
| dirty. |
| <ul> |
| <li>We re-execute Q and compare the hash of its result to the hash of the |
| result from the previous compilation.</li> |
| <li>If the hash has not changed, we can mark Q as <strong>green</strong> and return.</li> |
| </ul> |
| </li> |
| <li>Otherwise, <strong>all</strong> of the nodes in <code>reads(Q)</code> must be <strong>green</strong>. In that |
| case, we can color Q as <strong>green</strong> and return.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p><a id="dag"></a></p> |
| <h3 id="the-query-dag"><a class="header" href="#the-query-dag">The query DAG</a></h3> |
| <p>The query DAG code is stored in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/dep_graph/index.html"><code>compiler/rustc_middle/src/dep_graph</code></a>. Construction of the DAG is done |
| by instrumenting the query execution.</p> |
| <p>One key point is that the query DAG also tracks ordering; that is, for |
| each query Q, we not only track the queries that Q reads, we track the |
| <strong>order</strong> in which they were read. This allows try-mark-green to walk |
| those queries back in the same order. This is important because once a |
| subquery comes back as red, we can no longer be sure that Q will continue |
| along the same path as before. That is, imagine a query like this:</p> |
| <pre><code class="language-rust ignore">fn main_query(tcx) { |
| if tcx.subquery1() { |
| tcx.subquery2() |
| } else { |
| tcx.subquery3() |
| } |
| }</code></pre> |
| <p>Now imagine that in the first compilation, <code>main_query</code> starts by |
| executing <code>subquery1</code>, and this returns true. In that case, the next |
| query <code>main_query</code> executes will be <code>subquery2</code>, and <code>subquery3</code> will |
| not be executed at all.</p> |
| <p>But now imagine that in the <strong>next</strong> compilation, the input has |
| changed such that <code>subquery1</code> returns <strong>false</strong>. In this case, <code>subquery2</code> |
| would never execute. If try-mark-green were to visit <code>reads(main_query)</code> out |
| of order, however, it might visit <code>subquery2</code> before <code>subquery1</code>, and hence |
| execute it. |
| This can lead to ICEs and other problems in the compiler.</p> |
| <h2 id="improvements-to-the-basic-algorithm"><a class="header" href="#improvements-to-the-basic-algorithm">Improvements to the basic algorithm</a></h2> |
| <p>In the description of the basic algorithm, we said that at the end of |
| compilation we would save the results of all the queries that were |
| performed. In practice, this can be quite wasteful – many of those |
| results are very cheap to recompute, and serializing and deserializing |
| them is not a particular win. In practice, what we would do is to save |
| <strong>the hashes</strong> of all the subqueries that we performed. Then, in select cases, |
| we <strong>also</strong> save the results.</p> |
| <p>This is why the incremental algorithm separates computing the |
| <strong>color</strong> of a node, which often does not require its value, from |
| computing the <strong>result</strong> of a node. Computing the result is done via a simple |
| algorithm like so:</p> |
| <ul> |
| <li>Check if a saved result for Q is available. If so, compute the color of Q. |
| If Q is green, deserialize and return the saved result.</li> |
| <li>Otherwise, execute Q. |
| <ul> |
| <li>We can then compare the hash of the result and color Q as green if |
| it did not change.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="resources"><a class="header" href="#resources">Resources</a></h2> |
| <p>The initial design document can be found <a href="https://github.com/nikomatsakis/rustc-on-demand-incremental-design-doc/blob/master/0000-rustc-on-demand-and-incremental.md">here</a>, which expands |
| on the memoization details, provides more high-level overview and motivation |
| for this system.</p> |
| <h1 id="footnotes"><a class="header" href="#footnotes">Footnotes</a></h1> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-salsa"> |
| <p>I have long wanted to rename it to the Salsa algorithm, but it never caught on. -@nikomatsakis <a href="#fr-salsa-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="incremental-compilation-in-detail"><a class="header" href="#incremental-compilation-in-detail">Incremental compilation in detail</a></h1> |
| <p>The incremental compilation scheme is, in essence, a surprisingly |
| simple extension to the overall query system. It relies on the fact that:</p> |
| <ol> |
| <li>queries are pure functions -- given the same inputs, a query will always |
| yield the same result, and</li> |
| <li>the query model structures compilation in an acyclic graph that makes |
| dependencies between individual computations explicit.</li> |
| </ol> |
| <p>This chapter will explain how we can use these properties for making things |
| incremental and then goes on to discuss version implementation issues.</p> |
| <h2 id="a-basic-algorithm-for-incremental-query-evaluation"><a class="header" href="#a-basic-algorithm-for-incremental-query-evaluation">A Basic Algorithm For Incremental Query Evaluation</a></h2> |
| <p>As explained in the <a href="queries/./query-evaluation-model-in-detail.html">query evaluation model primer</a>, query |
| invocations form a directed-acyclic graph. Here's the example from the |
| previous chapter again:</p> |
| <pre><code class="language-ignore"> list_of_all_hir_items <----------------------------- type_check_crate() |
| | |
| | |
| Hir(foo) <--- type_of(foo) <--- type_check_item(foo) <-------+ |
| | | |
| +-----------------+ | |
| | | |
| v | |
| Hir(bar) <--- type_of(bar) <--- type_check_item(bar) <-------+ |
| </code></pre> |
| <p>Since every access from one query to another has to go through the query |
| context, we can record these accesses and thus actually build this dependency |
| graph in memory. With dependency tracking enabled, when compilation is done, |
| we know which queries were invoked (the nodes of the graph) and for each |
| invocation, which other queries or input has gone into computing the query's |
| result (the edges of the graph).</p> |
| <p>Now suppose we change the source code of our program so that |
| HIR of <code>bar</code> looks different than before. Our goal is to only recompute |
| those queries that are actually affected by the change while re-using |
| the cached results of all the other queries. Given the dependency graph we can |
| do exactly that. For a given query invocation, the graph tells us exactly |
| what data has gone into computing its results, we just have to follow the |
| edges until we reach something that has changed. If we don't encounter |
| anything that has changed, we know that the query still would evaluate to |
| the same result we already have in our cache.</p> |
| <p>Taking the <code>type_of(foo)</code> invocation from above as an example, we can check |
| whether the cached result is still valid by following the edges to its |
| inputs. The only edge leads to <code>Hir(foo)</code>, an input that has not been affected |
| by the change. So we know that the cached result for <code>type_of(foo)</code> is still |
| valid.</p> |
| <p>The story is a bit different for <code>type_check_item(foo)</code>: We again walk the |
| edges and already know that <code>type_of(foo)</code> is fine. Then we get to |
| <code>type_of(bar)</code> which we have not checked yet, so we walk the edges of |
| <code>type_of(bar)</code> and encounter <code>Hir(bar)</code> which <em>has</em> changed. Consequently |
| the result of <code>type_of(bar)</code> might yield a different result than what we |
| have in the cache and, transitively, the result of <code>type_check_item(foo)</code> |
| might have changed too. We thus re-run <code>type_check_item(foo)</code>, which in |
| turn will re-run <code>type_of(bar)</code>, which will yield an up-to-date result |
| because it reads the up-to-date version of <code>Hir(bar)</code>. Also, we re-run |
| <code>type_check_item(bar)</code> because result of <code>type_of(bar)</code> might have changed.</p> |
| <h2 id="the-problem-with-the-basic-algorithm-false-positives"><a class="header" href="#the-problem-with-the-basic-algorithm-false-positives">The problem with the basic algorithm: false positives</a></h2> |
| <p>If you read the previous paragraph carefully you'll notice that it says that |
| <code>type_of(bar)</code> <em>might</em> have changed because one of its inputs has changed. |
| There's also the possibility that it might still yield exactly the same |
| result <em>even though</em> its input has changed. Consider an example with a |
| simple query that just computes the sign of an integer:</p> |
| <pre><code class="language-ignore"> IntValue(x) <---- sign_of(x) <--- some_other_query(x) |
| </code></pre> |
| <p>Let's say that <code>IntValue(x)</code> starts out as <code>1000</code> and then is set to <code>2000</code>. |
| Even though <code>IntValue(x)</code> is different in the two cases, <code>sign_of(x)</code> yields |
| the result <code>+</code> in both cases.</p> |
| <p>If we follow the basic algorithm, however, <code>some_other_query(x)</code> would have to |
| (unnecessarily) be re-evaluated because it transitively depends on a changed |
| input. Change detection yields a "false positive" in this case because it has |
| to conservatively assume that <code>some_other_query(x)</code> might be affected by that |
| changed input.</p> |
| <p>Unfortunately it turns out that the actual queries in the compiler are full |
| of examples like this and small changes to the input often potentially affect |
| very large parts of the output binaries. As a consequence, we had to make the |
| change detection system smarter and more accurate.</p> |
| <h2 id="improving-accuracy-the-red-green-algorithm"><a class="header" href="#improving-accuracy-the-red-green-algorithm">Improving accuracy: the red-green algorithm</a></h2> |
| <p>The "false positives" problem can be solved by interleaving change detection |
| and query re-evaluation. Instead of walking the graph all the way to the |
| inputs when trying to find out if some cached result is still valid, we can |
| check if a result has <em>actually</em> changed after we were forced to re-evaluate |
| it.</p> |
| <p>We call this algorithm the red-green algorithm because nodes |
| in the dependency graph are assigned the color green if we were able to prove |
| that its cached result is still valid and the color red if the result has |
| turned out to be different after re-evaluating it.</p> |
| <p>The meat of red-green change tracking is implemented in the try-mark-green |
| algorithm, that, you've guessed it, tries to mark a given node as green:</p> |
| <pre><code class="language-rust ignore">fn try_mark_green(tcx, current_node) -> bool { |
| |
| // Fetch the inputs to `current_node`, i.e. get the nodes that the direct |
| // edges from `node` lead to. |
| let dependencies = tcx.dep_graph.get_dependencies_of(current_node); |
| |
| // Now check all the inputs for changes |
| for dependency in dependencies { |
| |
| match tcx.dep_graph.get_node_color(dependency) { |
| Green => { |
| // This input has already been checked before and it has not |
| // changed; so we can go on to check the next one |
| } |
| Red => { |
| // We found an input that has changed. We cannot mark |
| // `current_node` as green without re-running the |
| // corresponding query. |
| return false |
| } |
| Unknown => { |
| // This is the first time we look at this node. Let's try |
| // to mark it green by calling try_mark_green() recursively. |
| if try_mark_green(tcx, dependency) { |
| // We successfully marked the input as green, on to the |
| // next. |
| } else { |
| // We could *not* mark the input as green. This means we |
| // don't know if its value has changed. In order to find |
| // out, we re-run the corresponding query now! |
| tcx.run_query_for(dependency); |
| |
| // Fetch and check the node color again. Running the query |
| // has forced it to either red (if it yielded a different |
| // result than we have in the cache) or green (if it |
| // yielded the same result). |
| match tcx.dep_graph.get_node_color(dependency) { |
| Red => { |
| // The input turned out to be red, so we cannot |
| // mark `current_node` as green. |
| return false |
| } |
| Green => { |
| // Re-running the query paid off! The result is the |
| // same as before, so this particular input does |
| // not invalidate `current_node`. |
| } |
| Unknown => { |
| // There is no way a node has no color after |
| // re-running the query. |
| panic!("unreachable") |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| // If we have gotten through the entire loop, it means that all inputs |
| // have turned out to be green. If all inputs are unchanged, it means |
| // that the query result corresponding to `current_node` cannot have |
| // changed either. |
| tcx.dep_graph.mark_green(current_node); |
| |
| true |
| }</code></pre> |
| <blockquote> |
| <p>NOTE: |
| The actual implementation can be found in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_query_system/dep_graph/graph.rs.html"><code>compiler/rustc_query_system/src/dep_graph/graph.rs</code></a></p> |
| </blockquote> |
| <p>By using red-green marking we can avoid the devastating cumulative effect of |
| having false positives during change detection. Whenever a query is executed |
| in incremental mode, we first check if its already green. If not, we run |
| <code>try_mark_green()</code> on it. If it still isn't green after that, then we actually |
| invoke the query provider to re-compute the result. Re-computing the query might |
| then itself involve recursively invoking more queries, which can mean we come back |
| to the <code>try_mark_green()</code> algorithm for the dependencies recursively.</p> |
| <h2 id="the-real-world-how-persistence-makes-everything-complicated"><a class="header" href="#the-real-world-how-persistence-makes-everything-complicated">The real world: how persistence makes everything complicated</a></h2> |
| <p>The sections above described the underlying algorithm for incremental |
| compilation but because the compiler process exits after being finished and |
| takes the query context with its result cache with it into oblivion, we have to |
| persist data to disk, so the next compilation session can make use of it. |
| This comes with a whole new set of implementation challenges:</p> |
| <ul> |
| <li>The query result cache is stored to disk, so they are not readily available |
| for change comparison.</li> |
| <li>A subsequent compilation session will start off with new version of the code |
| that has arbitrary changes applied to it. All kinds of IDs and indices that |
| are generated from a global, sequential counter (e.g. <code>NodeId</code>, <code>DefId</code>, etc) |
| might have shifted, making the persisted results on disk not immediately |
| usable anymore because the same numeric IDs and indices might refer to |
| completely new things in the new compilation session.</li> |
| <li>Persisting things to disk comes at a cost, so not every tiny piece of |
| information should be actually cached in between compilation sessions. |
| Fixed-sized, plain-old-data is preferred to complex things that need to run |
| through an expensive (de-)serialization step.</li> |
| </ul> |
| <p>The following sections describe how the compiler solves these issues.</p> |
| <h3 id="a-question-of-stability-bridging-the-gap-between-compilation-sessions"><a class="header" href="#a-question-of-stability-bridging-the-gap-between-compilation-sessions">A Question Of Stability: Bridging The Gap Between Compilation Sessions</a></h3> |
| <p>As noted before, various IDs (like <code>DefId</code>) are generated by the compiler in a |
| way that depends on the contents of the source code being compiled. ID assignment |
| is usually deterministic, that is, if the exact same code is compiled twice, |
| the same things will end up with the same IDs. However, if something |
| changes, e.g. a function is added in the middle of a file, there is no |
| guarantee that anything will have the same ID as it had before.</p> |
| <p>As a consequence we cannot represent the data in our on-disk cache the same |
| way it is represented in memory. For example, if we just stored a piece |
| of type information like <code>TyKind::FnDef(DefId, &'tcx Substs<'tcx>)</code> (as we do |
| in memory) and then the contained <code>DefId</code> points to a different function in |
| a new compilation session we'd be in trouble.</p> |
| <p>The solution to this problem is to find "stable" forms for IDs which remain |
| valid in between compilation sessions. For the most important case, <code>DefId</code>s, |
| these are the so-called <code>DefPath</code>s. Each <code>DefId</code> has a |
| corresponding <code>DefPath</code> but in place of a numeric ID, a <code>DefPath</code> is based on |
| the path to the identified item, e.g. <code>std::collections::HashMap</code>. The |
| advantage of an ID like this is that it is not affected by unrelated changes. |
| For example, one can add a new function to <code>std::collections</code> but |
| <code>std::collections::HashMap</code> would still be <code>std::collections::HashMap</code>. A |
| <code>DefPath</code> is "stable" across changes made to the source code while a <code>DefId</code> |
| isn't.</p> |
| <p>There is also the <code>DefPathHash</code> which is just a 128-bit hash value of the |
| <code>DefPath</code>. The two contain the same information and we mostly use the |
| <code>DefPathHash</code> because it simpler to handle, being <code>Copy</code> and self-contained.</p> |
| <p>This principle of stable identifiers is used to make the data in the on-disk |
| cache resilient to source code changes. Instead of storing a <code>DefId</code>, we store |
| the <code>DefPathHash</code> and when we deserialize something from the cache, we map the |
| <code>DefPathHash</code> to the corresponding <code>DefId</code> in the <em>current</em> compilation session |
| (which is just a simple hash table lookup).</p> |
| <p>The <code>HirId</code>, used for identifying HIR components that don't have their own |
| <code>DefId</code>, is another such stable ID. It is (conceptually) a pair of a <code>DefPath</code> |
| and a <code>LocalId</code>, where the <code>LocalId</code> identifies something (e.g. a <code>hir::Expr</code>) |
| locally within its "owner" (e.g. a <code>hir::Item</code>). If the owner is moved around, |
| the <code>LocalId</code>s within it are still the same.</p> |
| <h3 id="checking-query-results-for-changes-hashstable-and-fingerprints"><a class="header" href="#checking-query-results-for-changes-hashstable-and-fingerprints">Checking query results for changes: <code>HashStable</code> and <code>Fingerprint</code>s</a></h3> |
| <p>In order to do red-green-marking we often need to check if the result of a |
| query has changed compared to the result it had during the previous |
| compilation session. There are two performance problems with this though:</p> |
| <ul> |
| <li>We'd like to avoid having to load the previous result from disk just for |
| doing the comparison. We already computed the new result and will use that. |
| Also loading a result from disk will "pollute" the interners with data that |
| is unlikely to ever be used.</li> |
| <li>We don't want to store each and every result in the on-disk cache. For |
| example, it would be wasted effort to persist things to disk that are |
| already available in upstream crates.</li> |
| </ul> |
| <p>The compiler avoids these problems by using so-called <code>Fingerprint</code>s. Each time |
| a new query result is computed, the query engine will compute a 128 bit hash |
| value of the result. We call this hash value "the <code>Fingerprint</code> of the query |
| result". The hashing is (and has to be) done "in a stable way". This means |
| that whenever something is hashed that might change in between compilation |
| sessions (e.g. a <code>DefId</code>), we instead hash its stable equivalent |
| (e.g. the corresponding <code>DefPath</code>). That's what the whole <code>HashStable</code> |
| infrastructure is for. This way <code>Fingerprint</code>s computed in two |
| different compilation sessions are still comparable.</p> |
| <p>The next step is to store these fingerprints along with the dependency graph. |
| This is cheap since fingerprints are just bytes to be copied. It's also cheap to |
| load the entire set of fingerprints together with the dependency graph.</p> |
| <p>Now, when red-green-marking reaches the point where it needs to check if a |
| result has changed, it can just compare the (already loaded) previous |
| fingerprint to the fingerprint of the new result.</p> |
| <p>This approach works rather well but it's not without flaws:</p> |
| <ul> |
| <li> |
| <p>There is a small possibility of hash collisions. That is, two different |
| results could have the same fingerprint and the system would erroneously |
| assume that the result hasn't changed, leading to a missed update.</p> |
| <p>We mitigate this risk by using a high-quality hash function and a 128 bit |
| wide hash value. Due to these measures the practical risk of a hash |
| collision is negligible.</p> |
| </li> |
| <li> |
| <p>Computing fingerprints is quite costly. It is the main reason why incremental |
| compilation can be slower than non-incremental compilation. We are forced to |
| use a good and thus expensive hash function, and we have to map things to |
| their stable equivalents while doing the hashing.</p> |
| </li> |
| </ul> |
| <h3 id="a-tale-of-two-depgraphs-the-old-and-the-new"><a class="header" href="#a-tale-of-two-depgraphs-the-old-and-the-new">A tale of two <code>DepGraph</code>s: the old and the new</a></h3> |
| <p>The initial description of dependency tracking glosses over a few details |
| that quickly become a head scratcher when actually trying to implement things. |
| In particular it's easy to overlook that we are actually dealing with <em>two</em> |
| dependency graphs: The one we built during the previous compilation session and |
| the one that we are building for the current compilation session.</p> |
| <p>When a compilation session starts, the compiler loads the previous dependency |
| graph into memory as an immutable piece of data. Then, when a query is invoked, |
| it will first try to mark the corresponding node in the graph as green. This |
| means really that we are trying to mark the node in the <em>previous</em> dep-graph |
| as green that corresponds to the query key in the <em>current</em> session. How do we |
| do this mapping between current query key and previous <code>DepNode</code>? The answer |
| is again <code>Fingerprint</code>s: Nodes in the dependency graph are identified by a |
| fingerprint of the query key. Since fingerprints are stable across compilation |
| sessions, computing one in the current session allows us to find a node |
| in the dependency graph from the previous session. If we don't find a node with |
| the given fingerprint, it means that the query key refers to something that |
| did not yet exist in the previous session.</p> |
| <p>So, having found the dep-node in the previous dependency graph, we can look |
| up its dependencies (i.e. also dep-nodes in the previous graph) and continue with |
| the rest of the try-mark-green algorithm. The next interesting thing happens |
| when we successfully marked the node as green. At that point we copy the node |
| and the edges to its dependencies from the old graph into the new graph. We |
| have to do this because the new dep-graph cannot acquire the |
| node and edges via the regular dependency tracking. The tracking system can |
| only record edges while actually running a query -- but running the query, |
| although we have the result already cached, is exactly what we want to avoid.</p> |
| <p>Once the compilation session has finished, all the unchanged parts have been |
| copied over from the old into the new dependency graph, while the changed parts |
| have been added to the new graph by the tracking system. At this point, the |
| new graph is serialized out to disk, alongside the query result cache, and can |
| act as the previous dep-graph in a subsequent compilation session.</p> |
| <h3 id="didnt-you-forget-something-cache-promotion"><a class="header" href="#didnt-you-forget-something-cache-promotion">Didn't you forget something?: cache promotion</a></h3> |
| <p>The system described so far has a somewhat subtle property: If all inputs of a |
| dep-node are green then the dep-node itself can be marked as green without |
| computing or loading the corresponding query result. Applying this property |
| transitively often leads to the situation that some intermediate results are |
| never actually loaded from disk, as in the following example:</p> |
| <pre><code class="language-ignore"> input(A) <-- intermediate_query(B) <-- leaf_query(C) |
| </code></pre> |
| <p>The compiler might need the value of <code>leaf_query(C)</code> in order to generate some |
| output artifact. If it can mark <code>leaf_query(C)</code> as green, it will load the |
| result from the on-disk cache. The result of <code>intermediate_query(B)</code> is never |
| loaded though. As a consequence, when the compiler persists the <em>new</em> result |
| cache by writing all in-memory query results to disk, <code>intermediate_query(B)</code> |
| will not be in memory and thus will be missing from the new result cache.</p> |
| <p>If there subsequently is another compilation session that actually needs the |
| result of <code>intermediate_query(B)</code> it will have to be re-computed even though we |
| had a perfectly valid result for it in the cache just before.</p> |
| <p>In order to prevent this from happening, the compiler does something called |
| "cache promotion": Before emitting the new result cache it will walk all green |
| dep-nodes and make sure that their query result is loaded into memory. That way |
| the result cache doesn't unnecessarily shrink again.</p> |
| <h1 id="incremental-compilation-and-the-compiler-backend"><a class="header" href="#incremental-compilation-and-the-compiler-backend">Incremental compilation and the compiler backend</a></h1> |
| <p>The compiler backend, the part involving LLVM, is using the query system but |
| it is not implemented in terms of queries itself. As a consequence it does not |
| automatically partake in dependency tracking. However, the manual integration |
| with the tracking system is pretty straight-forward. The compiler simply tracks |
| what queries get invoked when generating the initial LLVM version of each |
| codegen unit (CGU), which results in a dep-node for each CGU. In subsequent |
| compilation sessions it then tries to mark the dep-node for a CGU as green. If |
| it succeeds, it knows that the corresponding object and bitcode files on disk |
| are still valid. If it doesn't succeed, the entire CGU has to be recompiled.</p> |
| <p>This is the same approach that is used for regular queries. The main differences |
| are:</p> |
| <ul> |
| <li> |
| <p>that we cannot easily compute a fingerprint for LLVM modules (because |
| they are opaque C++ objects),</p> |
| </li> |
| <li> |
| <p>that the logic for dealing with cached values is rather different from |
| regular queries because here we have bitcode and object files instead of |
| serialized Rust values in the common result cache file, and</p> |
| </li> |
| <li> |
| <p>the operations around LLVM are so expensive in terms of computation time and |
| memory consumption that we need to have tight control over what is |
| executed when and what stays in memory for how long.</p> |
| </li> |
| </ul> |
| <p>The query system could probably be extended with general purpose mechanisms to |
| deal with all of the above but so far that seemed like more trouble than it |
| would save.</p> |
| <h2 id="query-modifiers"><a class="header" href="#query-modifiers">Query modifiers</a></h2> |
| <p>The query system allows for applying <a href="queries/../query.html#adding-a-new-kind-of-query">modifiers</a> to queries. These |
| modifiers affect certain aspects of how the system treats the query with |
| respect to incremental compilation:</p> |
| <ul> |
| <li> |
| <p><code>eval_always</code> - A query with the <code>eval_always</code> attribute is re-executed |
| unconditionally during incremental compilation. I.e. the system will not |
| even try to mark the query's dep-node as green. This attribute has two use |
| cases:</p> |
| <ul> |
| <li> |
| <p><code>eval_always</code> queries can read inputs (from files, global state, etc). |
| They can also produce side effects like writing to files and changing global state.</p> |
| </li> |
| <li> |
| <p>Some queries are very likely to be re-evaluated because their result |
| depends on the entire source code. In this case <code>eval_always</code> can be used |
| as an optimization because the system can skip recording dependencies in |
| the first place.</p> |
| </li> |
| </ul> |
| </li> |
| <li> |
| <p><code>no_hash</code> - Applying <code>no_hash</code> to a query tells the system to not compute |
| the fingerprint of the query's result. This has two consequences:</p> |
| <ul> |
| <li> |
| <p>Not computing the fingerprint can save quite a bit of time because |
| fingerprinting is expensive, especially for large, complex values.</p> |
| </li> |
| <li> |
| <p>Without the fingerprint, the system has to unconditionally assume that |
| the result of the query has changed. As a consequence anything depending |
| on a <code>no_hash</code> query will always be re-executed.</p> |
| </li> |
| </ul> |
| <p>Using <code>no_hash</code> for a query can make sense in two circumstances:</p> |
| <ul> |
| <li> |
| <p>If the result of the query is very likely to change whenever one of its |
| inputs changes, e.g. a function like <code>|a, b, c| -> (a * b * c)</code>. In such |
| a case recomputing the query will always yield a red node if one of the |
| inputs is red so we can spare us the trouble and default to red immediately. |
| A counter example would be a function like <code>|a| -> (a == 42)</code> where the |
| result does not change for most changes of <code>a</code>.</p> |
| </li> |
| <li> |
| <p>If the result of a query is a big, monolithic collection (e.g. <code>index_hir</code>) |
| and there are "projection queries" reading from that collection |
| (e.g. <code>hir_owner</code>). In such a case the big collection will likely fulfill the |
| condition above (any changed input means recomputing the whole collection) |
| and the results of the projection queries will be hashed anyway. If we also |
| hashed the collection query it would mean that we effectively hash the same |
| data twice: once when hashing the collection and another time when hashing all |
| the projection query results. <code>no_hash</code> allows us to avoid that redundancy |
| and the projection queries act as a "firewall", shielding their dependents |
| from the unconditionally red <code>no_hash</code> node.</p> |
| </li> |
| </ul> |
| </li> |
| <li> |
| <p><code>cache_on_disk_if</code> - This attribute is what determines which query results |
| are persisted in the incremental compilation query result cache. The |
| attribute takes an expression that allows per query invocation |
| decisions. For example, it makes no sense to store values from upstream |
| crates in the cache because they are already available in the upstream |
| crate's metadata.</p> |
| </li> |
| <li> |
| <p><code>anon</code> - This attribute makes the system use "anonymous" dep-nodes for the |
| given query. An anonymous dep-node is not identified by the corresponding |
| query key, instead its ID is computed from the IDs of its dependencies. This |
| allows the red-green system to do its change detection even if there is no |
| query key available for a given dep-node -- something which is needed for |
| handling trait selection because it is not based on queries.</p> |
| </li> |
| </ul> |
| <h2 id="the-projection-query-pattern"><a class="header" href="#the-projection-query-pattern">The projection query pattern</a></h2> |
| <p>It's interesting to note that <code>eval_always</code> and <code>no_hash</code> can be used together |
| in the so-called "projection query" pattern. It is often the case that there is |
| one query that depends on the entirety of the compiler's input (e.g. the indexed HIR) |
| and another query that projects individual values out of this monolithic value |
| (e.g. a HIR item with a certain <code>DefId</code>). These projection queries allow for |
| building change propagation "firewalls" because even if the result of the |
| monolithic query changes (which it is very likely to do) the small projections |
| can still mostly be marked as green.</p> |
| <pre><code class="language-ignore"> +------------+ |
| | | +---------------+ +--------+ |
| | | <---------| projection(x) | <---------| foo(a) | |
| | | +---------------+ +--------+ |
| | | |
| | monolithic | +---------------+ +--------+ |
| | query | <---------| projection(y) | <---------| bar(b) | |
| | | +---------------+ +--------+ |
| | | |
| | | +---------------+ +--------+ |
| | | <---------| projection(z) | <---------| baz(c) | |
| | | +---------------+ +--------+ |
| +------------+ |
| </code></pre> |
| <p>Let's assume that the result <code>monolithic_query</code> changes so that also the result |
| of <code>projection(x)</code> has changed, i.e. both their dep-nodes are being marked as |
| red. As a consequence <code>foo(a)</code> needs to be re-executed; but <code>bar(b)</code> and |
| <code>baz(c)</code> can be marked as green. However, if <code>foo</code>, <code>bar</code>, and <code>baz</code> would have |
| directly depended on <code>monolithic_query</code> then all of them would have had to be |
| re-evaluated.</p> |
| <p>This pattern works even without <code>eval_always</code> and <code>no_hash</code> but the two |
| modifiers can be used to avoid unnecessary overhead. If the monolithic query |
| is likely to change at any minor modification of the compiler's input it makes |
| sense to mark it as <code>eval_always</code>, thus getting rid of its dependency tracking |
| cost. And it always makes sense to mark the monolithic query as <code>no_hash</code> |
| because we have the projections to take care of keeping things green as much |
| as possible.</p> |
| <h1 id="shortcomings-of-the-current-system"><a class="header" href="#shortcomings-of-the-current-system">Shortcomings of the current system</a></h1> |
| <p>There are many things that still can be improved.</p> |
| <h2 id="incrementality-of-on-disk-data-structures"><a class="header" href="#incrementality-of-on-disk-data-structures">Incrementality of on-disk data structures</a></h2> |
| <p>The current system is not able to update on-disk caches and the dependency graph |
| in-place. Instead it has to rewrite each file entirely in each compilation |
| session. The overhead of doing so is a few percent of total compilation time.</p> |
| <h2 id="unnecessary-data-dependencies"><a class="header" href="#unnecessary-data-dependencies">Unnecessary data dependencies</a></h2> |
| <p>Data structures used as query results could be factored in a way that removes |
| edges from the dependency graph. Especially "span" information is very volatile, |
| so including it in query result will increase the chance that the result won't |
| be reusable. See <a href="https://github.com/rust-lang/rust/issues/47389">https://github.com/rust-lang/rust/issues/47389</a> for more |
| information.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="debugging-and-testing-dependencies"><a class="header" href="#debugging-and-testing-dependencies">Debugging and testing dependencies</a></h1> |
| <h2 id="testing-the-dependency-graph"><a class="header" href="#testing-the-dependency-graph">Testing the dependency graph</a></h2> |
| <p>There are various ways to write tests against the dependency graph. The |
| simplest mechanisms are the <code>#[rustc_if_this_changed]</code> and |
| <code>#[rustc_then_this_would_need]</code> annotations. These are used in <a href="tests/ui.html">ui</a> tests to test |
| whether the expected set of paths exist in the dependency graph.</p> |
| <p>As an example, see <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui/dep-graph/dep-graph-caller-callee.rs"><code>tests/ui/dep-graph/dep-graph-caller-callee.rs</code></a>, or the |
| tests below.</p> |
| <pre><code class="language-rust ignore">#[rustc_if_this_changed] |
| fn foo() { } |
| |
| #[rustc_then_this_would_need(TypeckTables)] //~ ERROR OK |
| fn bar() { foo(); }</code></pre> |
| <p>This should be read as</p> |
| <blockquote> |
| <p>If this (<code>foo</code>) is changed, then this (i.e. <code>bar</code>)'s TypeckTables would need to be changed.</p> |
| </blockquote> |
| <p>Technically, what occurs is that the test is expected to emit the string "OK" on |
| stderr, associated to this line.</p> |
| <p>You could also add the lines</p> |
| <pre><code class="language-rust ignore">#[rustc_then_this_would_need(TypeckTables)] //~ ERROR no path |
| fn baz() { }</code></pre> |
| <p>Whose meaning is</p> |
| <blockquote> |
| <p>If <code>foo</code> is changed, then <code>baz</code>'s TypeckTables does not need to be changed. |
| The macro must emit an error, and the error message must contains "no path".</p> |
| </blockquote> |
| <p>Recall that the <code>//~ ERROR OK</code> is a comment from the point of view of the Rust |
| code we test, but is meaningful from the point of view of the test itself.</p> |
| <h2 id="debugging-the-dependency-graph"><a class="header" href="#debugging-the-dependency-graph">Debugging the dependency graph</a></h2> |
| <h3 id="dumping-the-graph"><a class="header" href="#dumping-the-graph">Dumping the graph</a></h3> |
| <p>The compiler is also capable of dumping the dependency graph for your |
| debugging pleasure. To do so, pass the <code>-Z dump-dep-graph</code> flag. The |
| graph will be dumped to <code>dep_graph.{txt,dot}</code> in the current |
| directory. You can override the filename with the <code>RUST_DEP_GRAPH</code> |
| environment variable.</p> |
| <p>Frequently, though, the full dep graph is quite overwhelming and not |
| particularly helpful. Therefore, the compiler also allows you to filter |
| the graph. You can filter in three ways:</p> |
| <ol> |
| <li>All edges originating in a particular set of nodes (usually a single node).</li> |
| <li>All edges reaching a particular set of nodes.</li> |
| <li>All edges that lie between given start and end nodes.</li> |
| </ol> |
| <p>To filter, use the <code>RUST_DEP_GRAPH_FILTER</code> environment variable, which should |
| look like one of the following:</p> |
| <pre><code class="language-text">source_filter // nodes originating from source_filter |
| -> target_filter // nodes that can reach target_filter |
| source_filter -> target_filter // nodes in between source_filter and target_filter |
| </code></pre> |
| <p><code>source_filter</code> and <code>target_filter</code> are a <code>&</code>-separated list of strings. |
| A node is considered to match a filter if all of those strings appear in its |
| label. So, for example:</p> |
| <pre><code class="language-text">RUST_DEP_GRAPH_FILTER='-> TypeckTables' |
| </code></pre> |
| <p>would select the predecessors of all <code>TypeckTables</code> nodes. Usually though you |
| want the <code>TypeckTables</code> node for some particular fn, so you might write:</p> |
| <pre><code class="language-text">RUST_DEP_GRAPH_FILTER='-> TypeckTables & bar' |
| </code></pre> |
| <p>This will select only the predecessors of <code>TypeckTables</code> nodes for functions |
| with <code>bar</code> in their name.</p> |
| <p>Perhaps you are finding that when you change <code>foo</code> you need to re-type-check |
| <code>bar</code>, but you don't think you should have to. In that case, you might do:</p> |
| <pre><code class="language-text">RUST_DEP_GRAPH_FILTER='Hir & foo -> TypeckTables & bar' |
| </code></pre> |
| <p>This will dump out all the nodes that lead from <code>Hir(foo)</code> to |
| <code>TypeckTables(bar)</code>, from which you can (hopefully) see the source |
| of the erroneous edge.</p> |
| <h3 id="tracking-down-incorrect-edges"><a class="header" href="#tracking-down-incorrect-edges">Tracking down incorrect edges</a></h3> |
| <p>Sometimes, after you dump the dependency graph, you will find some |
| path that should not exist, but you will not be quite sure how it came |
| to be. <strong>When the compiler is built with debug assertions,</strong> it can |
| help you track that down. Simply set the <code>RUST_FORBID_DEP_GRAPH_EDGE</code> |
| environment variable to a filter. Every edge created in the dep-graph |
| will be tested against that filter – if it matches, a <code>bug!</code> is |
| reported, so you can easily see the backtrace (<code>RUST_BACKTRACE=1</code>).</p> |
| <p>The syntax for these filters is the same as described in the previous |
| section. However, note that this filter is applied to every <strong>edge</strong> |
| and doesn't handle longer paths in the graph, unlike the previous |
| section.</p> |
| <p>Example:</p> |
| <p>You find that there is a path from the <code>Hir</code> of <code>foo</code> to the type |
| check of <code>bar</code> and you don't think there should be. You dump the |
| dep-graph as described in the previous section and open <code>dep-graph.txt</code> |
| to see something like:</p> |
| <pre><code class="language-text">Hir(foo) -> Collect(bar) |
| Collect(bar) -> TypeckTables(bar) |
| </code></pre> |
| <p>That first edge looks suspicious to you. So you set |
| <code>RUST_FORBID_DEP_GRAPH_EDGE</code> to <code>Hir&foo -> Collect&bar</code>, re-run, and |
| then observe the backtrace. Voila, bug fixed!</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="how-salsa-works"><a class="header" href="#how-salsa-works">How Salsa works</a></h1> |
| <p>This chapter is based on the explanation given by Niko Matsakis in this |
| <a href="https://www.youtube.com/watch?v=_muY4HjSqVw">video</a> about |
| <a href="https://github.com/salsa-rs/salsa">Salsa</a>. To find out more you may |
| want to watch <a href="https://www.youtube.com/watch?v=i_IhACacPRY">Salsa In More |
| Depth</a>, also by Niko |
| Matsakis.</p> |
| <blockquote> |
| <p>As of <!-- date-check --> November 2022, although Salsa is inspired by (among |
| other things) rustc's query system, it is not used directly in rustc. It |
| <em>is</em> used in <a href="https://rust-lang.github.io/chalk/book/what_is_chalk.html">chalk</a>, an implementation of Rust's trait system, and |
| extensively in <a href="https://rust-analyzer.github.io/"><code>rust-analyzer</code></a>, the official implementation of the language |
| server protocol for Rust, but there are no medium or long-term concrete |
| plans to integrate it into the compiler.</p> |
| </blockquote> |
| <h2 id="what-is-salsa"><a class="header" href="#what-is-salsa">What is Salsa?</a></h2> |
| <p>Salsa is a library for incremental recomputation. This means it allows reusing |
| computations that were already done in the past to increase the efficiency |
| of future computations.</p> |
| <p>The objectives of Salsa are:</p> |
| <ul> |
| <li>Provide that functionality in an automatic way, so reusing old computations |
| is done automatically by the library.</li> |
| <li>Doing so in a "sound", or "correct", way, therefore leading to the same |
| results as if it had been done from scratch.</li> |
| </ul> |
| <p>Salsa's actual model is much richer, allowing many kinds of inputs and many different outputs. |
| For example, integrating Salsa with an IDE could mean that |
| the inputs could be manifests (<code>Cargo.toml</code>, <code>rust-toolchain.toml</code>), entire |
| source files (<code>foo.rs</code>), snippets and so on. The outputs of such an integration |
| could range from a binary executable, to lints, types (for example, if a user |
| selects a certain variable and wishes to see its type), completions, etc.</p> |
| <h2 id="how-does-it-work"><a class="header" href="#how-does-it-work">How does it work?</a></h2> |
| <p>The first thing that Salsa has to do is identify the "base inputs" that |
| are not something computed but given as input.</p> |
| <p>Then Salsa has to also identify intermediate, "derived" values, which are |
| something that the library produces, but, for each derived value there's a |
| "pure" function that computes the derived value.</p> |
| <p>For example, there might be a function <code>ast(x: Path) -> AST</code>. The produced |
| Abstract Syntax Tree (<code>AST</code>) isn't a final value, it's an intermediate value |
| that the library would use for the computation.</p> |
| <p>This means that when you try to compute with the library, Salsa is going to |
| compute various derived values, and eventually read the input and produce the |
| result for the asked computation.</p> |
| <p>In the course of computing, Salsa tracks which inputs were accessed and which |
| values are derived. This information is used to determine what's going to |
| happen when the inputs change: are the derived values still valid?</p> |
| <p>This doesn't necessarily mean that each computation downstream from the input |
| is going to be checked, which could be costly. Salsa only needs to check each |
| downstream computation until it finds one that isn't changed. At that point, it |
| won't check other derived computations since they wouldn't need to change.</p> |
| <p>It's helpful to think about this as a graph with nodes. Each derived value |
| has a dependency on other values, which could themselves be either base or |
| derived. Base values don't have a dependency.</p> |
| <pre><code class="language-ignore">I <- A <- C ... |
| | |
| J <- B <--+ |
| </code></pre> |
| <p>When an input <code>I</code> changes, the derived value <code>A</code> could change. The derived |
| value <code>B</code>, which does not depend on <code>I</code>, <code>A</code>, or any value derived from <code>A</code> or |
| <code>I</code>, is not subject to change. Therefore, Salsa can reuse the computation done |
| for <code>B</code> in the past, without having to compute it again.</p> |
| <p>The computation could also terminate early. Keeping the same graph as before, |
| say that input <code>I</code> has changed in some way (and input <code>J</code> hasn't), but when |
| computing <code>A</code> again, it's found that <code>A</code> hasn't changed from the previous |
| computation. This leads to an "early termination", because there's no need to |
| check if <code>C</code> needs to change, since both <code>C</code> direct inputs, <code>A</code> and <code>B</code>, |
| haven't changed.</p> |
| <h2 id="key-salsa-concepts"><a class="header" href="#key-salsa-concepts">Key Salsa concepts</a></h2> |
| <h3 id="query"><a class="header" href="#query">Query</a></h3> |
| <p>A query is some value that Salsa can access in the course of computation. Each |
| query can have a number of keys (from 0 to many), and all queries have a |
| result, akin to functions. <code>0-key</code> queries are called "input" queries.</p> |
| <h3 id="database"><a class="header" href="#database">Database</a></h3> |
| <p>The database is basically the context for the entire computation, it's meant to |
| store Salsa's internal state, all intermediate values for each query, and |
| anything else that the computation might need. The database must know all the |
| queries the library is going to do before it can be built, but they don't need |
| to be specified in the same place.</p> |
| <p>After the database is formed, it can be accessed with queries that are very |
| similar to functions. Since each query's result is stored in the database, when |
| a query is invoked <code>N</code>-times, it will return <code>N</code>-<strong>cloned</strong> results, without having |
| to recompute the query (unless the input has changed in such a way that it |
| warrants recomputation).</p> |
| <p>For each input query (<code>0-key</code>), a "set" method is generated, allowing the user to |
| change the output of such query, and trigger previous memoized values to be |
| potentially invalidated.</p> |
| <h3 id="query-groups"><a class="header" href="#query-groups">Query Groups</a></h3> |
| <p>A query group is a set of queries which have been defined together as a unit. |
| The database is formed by combining query groups. Query groups are akin to |
| "Salsa modules".</p> |
| <p>A set of queries in a query group are just a set of methods in a trait.</p> |
| <p>To create a query group a trait annotated with a specific attribute |
| (<code>#[salsa::query_group(...)]</code>) has to be created.</p> |
| <p>An argument must also be provided to said attribute as it will be used by Salsa |
| to create a <code>struct</code> to be used later when the database is created.</p> |
| <p>Example input query group:</p> |
| <pre><code class="language-rust ignore">/// This attribute will process this tree, produce this tree as output, and produce |
| /// a bunch of intermediate stuff that Salsa also uses. One of these things is a |
| /// "StorageStruct", whose name we have specified in the attribute. |
| /// |
| /// This query group is a bunch of **input** queries, that do not rely on any |
| /// derived input. |
| #[salsa::query_group(InputsStorage)] |
| pub trait Inputs { |
| /// This attribute (`#[salsa::input]`) indicates that this query is a base |
| /// input, therefore `set_manifest` is going to be auto-generated |
| #[salsa::input] |
| fn manifest(&self) -> Manifest; |
| |
| #[salsa::input] |
| fn source_text(&self, name: String) -> String; |
| }</code></pre> |
| <p>To create a <strong>derived</strong> query group, one must specify which other query groups |
| this one depends on by specifying them as supertraits, as seen in the following |
| example:</p> |
| <pre><code class="language-rust ignore">/// This query group is going to contain queries that depend on derived values. |
| /// A query group can access another query group's queries by specifying the |
| /// dependency as a supertrait. Query groups can be stacked as much as needed using |
| /// that pattern. |
| #[salsa::query_group(ParserStorage)] |
| pub trait Parser: Inputs { |
| /// This query `ast` is not an input query, it's a derived query this means |
| /// that a definition is necessary. |
| fn ast(&self, name: String) -> String; |
| }</code></pre> |
| <p>When creating a derived query the implementation of said query must be defined |
| outside the trait. The definition must take a database parameter as an <code>impl Trait</code> (or <code>dyn Trait</code>), where trait is the query group that the definition |
| belongs to, in addition to the other keys.</p> |
| <pre><code class="language-rust ignore">/// This is going to be the definition of the `ast` query in the `Parser` trait. |
| /// So, when the query `ast` is invoked, and it needs to be recomputed, Salsa is |
| /// going to call this function and it's going to give it the database as `impl |
| /// Parser`. The function doesn't need to be aware of all the queries of all the |
| /// query groups |
| fn ast(db: &impl Parser, name: String) -> String { |
| //! Note, `impl Parser` is used here but `dyn Parser` works just as well |
| /* code */ |
| ///By passing an `impl Parser`, this is allowed |
| let source_text = db.input_file(name); |
| /* do the actual parsing */ |
| return ast; |
| }</code></pre> |
| <p>Eventually, after all the query groups have been defined, the database can be |
| created by declaring a <code>struct</code>.</p> |
| <p>To specify which query groups are going to be part of the database an <code>attribute</code> |
| (<code>#[salsa::database(...)]</code>) must be added. The argument of said <code>attribute</code> is a |
| list of <code>identifiers</code>, specifying the query groups <strong>storages</strong>.</p> |
| <pre><code class="language-rust ignore">///This attribute specifies which query groups are going to be in the database |
| #[salsa::database(InputsStorage, ParserStorage)] |
| #[derive(Default)] //optional! |
| struct MyDatabase { |
| ///You also need this one field |
| runtime : salsa::Runtime<MyDatabase>, |
| } |
| ///And this trait has to be implemented |
| impl salsa::Database for MyDatabase { |
| fn salsa_runtime(&self) -> &salsa::Runtime<MyDatabase> { |
| &self.runtime |
| } |
| }</code></pre> |
| <p>Example usage:</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let db = MyDatabase::default(); |
| db.set_manifest(...); |
| db.set_source_text(...); |
| loop { |
| db.ast(...); //will reuse results |
| db.set_source_text(...); |
| } |
| }</code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="memory-management-in-rustc"><a class="header" href="#memory-management-in-rustc">Memory management in rustc</a></h1> |
| <p>Generally rustc tries to be pretty careful how it manages memory. |
| The compiler allocates <em>a lot</em> of data structures throughout compilation, |
| and if we are not careful, it will take a lot of time and space to do so.</p> |
| <p>One of the main way the compiler manages this is using <a href="https://en.wikipedia.org/wiki/Region-based_memory_management">arena</a>s and <a href="https://en.wikipedia.org/wiki/String_interning">interning</a>.</p> |
| <h2 id="arenas-and--interning"><a class="header" href="#arenas-and--interning">Arenas and Interning</a></h2> |
| <p>Since A LOT of data structures are created during compilation, for performance |
| reasons, we allocate them from a global memory pool. |
| Each are allocated once from a long-lived <em>arena</em>. |
| This is called <em>arena allocation</em>. |
| This system reduces allocations/deallocations of memory. |
| It also allows for easy comparison of types (more on types <a href="./ty.html">here</a>) for equality: |
| for each interned type <code>X</code>, we implemented <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html#implementations"><code>PartialEq</code> for X</a>, |
| so we can just compare pointers. |
| The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.CtxtInterners.html#structfield.arena"><code>CtxtInterners</code></a> type contains a bunch of maps of interned types and the arena itself.</p> |
| <h3 id="example-tytykind"><a class="header" href="#example-tytykind">Example: <code>ty::TyKind</code></a></h3> |
| <p>Taking the example of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/sty/type.TyKind.html"><code>ty::TyKind</code></a> which represents a type in the compiler (you |
| can read more <a href="./ty.html">here</a>). Each time we want to construct a type, the |
| compiler doesn’t naively allocate from the buffer. Instead, we check if that |
| type was already constructed. If it was, we just get the same pointer we had |
| before, otherwise we make a fresh pointer. With this schema if we want to know |
| if two types are the same, all we need to do is compare the pointers which is |
| efficient. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/sty/type.TyKind.html"><code>ty::TyKind</code></a> should never be constructed on the stack, and it would be unusable |
| if done so. |
| You always allocate them from this arena and you always intern them so they are |
| unique.</p> |
| <p>At the beginning of the compilation we make a buffer and each time we need to allocate a type we use |
| some of this memory buffer. If we run out of space we get another one. The lifetime of that buffer |
| is <code>'tcx</code>. Our types are tied to that lifetime, so when compilation finishes all the memory related |
| to that buffer is freed and our <code>'tcx</code> references would be invalid.</p> |
| <p>In addition to types, there are a number of other arena-allocated data structures that you can |
| allocate, and which are found in this module. Here are a few examples:</p> |
| <ul> |
| <li><a href="./ty_module/generic_arguments.html#the-genericargs-type"><code>GenericArgs</code></a>, allocated with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.mk_args"><code>mk_args</code></a> – this will intern a slice of types, often used |
| to specify the values to be substituted for generics args (e.g. <code>HashMap<i32, u32></code> would be |
| represented as a slice <code>&'tcx [tcx.types.i32, tcx.types.u32]</code>).</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TraitRef.html"><code>TraitRef</code></a>, typically passed by value – a <strong>trait reference</strong> consists of a reference to a trait |
| along with its various type parameters (including <code>Self</code>), like <code>i32: Display</code> (here, the def-id |
| would reference the <code>Display</code> trait, and the args would contain <code>i32</code>). Note that <code>def-id</code> is |
| defined and discussed in depth in the <a href="./ty_module/generic_arguments.html#adtdef-and-defid"><code>AdtDef and DefId</code></a> section.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Predicate.html"><code>Predicate</code></a> defines something the trait system has to prove (see <a href="./traits/resolution.html">traits</a> module).</li> |
| </ul> |
| <h2 id="the-tcx-and-how-it-uses-lifetimes"><a class="header" href="#the-tcx-and-how-it-uses-lifetimes">The <code>tcx</code> and how it uses lifetimes</a></h2> |
| <p>The typing context (<code>tcx</code>) is the central data structure in the compiler. It is the context that |
| you use to perform all manner of queries. The <code>struct</code> <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a> defines a reference to this shared |
| context:</p> |
| <pre><code class="language-rust ignore">tcx: TyCtxt<'tcx> |
| // ---- |
| // | |
| // arena lifetime</code></pre> |
| <p>As you can see, the <code>TyCtxt</code> type takes a lifetime parameter. When you see a reference with a |
| lifetime like <code>'tcx</code>, you know that it refers to arena-allocated data (or data that lives as long as |
| the arenas, anyhow).</p> |
| <h3 id="a-note-on-lifetimes"><a class="header" href="#a-note-on-lifetimes">A Note On Lifetimes</a></h3> |
| <p>The Rust compiler is a fairly large program containing lots of big data |
| structures (e.g. the <a href="./ast-validation.html">Abstract Syntax Tree (AST)</a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/index.html">High-Level Intermediate |
| Representation (<code>HIR</code>)</a>, and the type system) and as such, arenas and |
| references are heavily relied upon to minimize unnecessary memory use. This |
| manifests itself in the way people can plug into the compiler (i.e. the |
| <a href="./rustc-driver/intro.html">driver</a>), preferring a "push"-style API (callbacks) instead |
| of the more Rust-ic "pull" style (think the <code>Iterator</code> trait).</p> |
| <p>Thread-local storage and interning are used a lot through the compiler to reduce |
| duplication while also preventing a lot of the ergonomic issues due to many |
| pervasive lifetimes. The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/tls/index.html"><code>rustc_middle::ty::tls</code></a> module is used to access these |
| thread-locals, although you should rarely need to touch it.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="serialization-in-rustc"><a class="header" href="#serialization-in-rustc">Serialization in rustc</a></h1> |
| <p>rustc has to <a href="https://en.wikipedia.org/wiki/Serialization">serialize</a> and deserialize various data during compilation. |
| Specifically:</p> |
| <ul> |
| <li>"Crate metadata", consisting mainly of query outputs, are serialized |
| from a binary format into <code>rlib</code> and <code>rmeta</code> files that are output when |
| compiling a library crate. These <code>rlib</code> and <code>rmeta</code> files are then |
| deserialized by the crates which depend on that library.</li> |
| <li>Certain query outputs are serialized in a binary format to |
| <a href="queries/incremental-compilation-in-detail.html#the-real-world-how-persistence-makes-everything-complicated">persist incremental compilation results</a>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/struct.CrateInfo.html"><code>CrateInfo</code></a> is serialized to <code>JSON</code> when the <code>-Z no-link</code> flag is used, and |
| deserialized from <code>JSON</code> when the <code>-Z link-only</code> flag is used.</li> |
| </ul> |
| <h2 id="the-encodable-and-decodable-traits"><a class="header" href="#the-encodable-and-decodable-traits">The <code>Encodable</code> and <code>Decodable</code> traits</a></h2> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_serialize/index.html"><code>rustc_serialize</code></a> crate defines two traits for types which can be serialized:</p> |
| <pre><code class="language-rust ignore">pub trait Encodable<S: Encoder> { |
| fn encode(&self, s: &mut S) -> Result<(), S::Error>; |
| } |
| |
| pub trait Decodable<D: Decoder>: Sized { |
| fn decode(d: &mut D) -> Result<Self, D::Error>; |
| }</code></pre> |
| <p>It also defines implementations of these for various common standard library |
| <a href="https://doc.rust-lang.org/std/#primitives">primitive types</a> such as integer |
| types, floating point types, <code>bool</code>, <code>char</code>, <code>str</code>, etc.</p> |
| <p>For types that are constructed from those types, <code>Encodable</code> and <code>Decodable</code> |
| are usually implemented by <a href="serialization.html#derive-macros">derives</a>. These generate implementations that |
| forward deserialization to the fields of the struct or enum. For a |
| struct those impls look something like this:</p> |
| <pre><code class="language-rust ignore">#![feature(rustc_private)] |
| extern crate rustc_serialize; |
| use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; |
| |
| struct MyStruct { |
| int: u32, |
| float: f32, |
| } |
| |
| impl<E: Encoder> Encodable<E> for MyStruct { |
| fn encode(&self, s: &mut E) -> Result<(), E::Error> { |
| s.emit_struct("MyStruct", 2, |s| { |
| s.emit_struct_field("int", 0, |s| self.int.encode(s))?; |
| s.emit_struct_field("float", 1, |s| self.float.encode(s)) |
| }) |
| } |
| } |
| |
| impl<D: Decoder> Decodable<D> for MyStruct { |
| fn decode(s: &mut D) -> Result<MyStruct, D::Error> { |
| s.read_struct("MyStruct", 2, |d| { |
| let int = d.read_struct_field("int", 0, Decodable::decode)?; |
| let float = d.read_struct_field("float", 1, Decodable::decode)?; |
| |
| Ok(MyStruct { int, float }) |
| }) |
| } |
| }</code></pre> |
| <h2 id="encoding-and-decoding-arena-allocated-types"><a class="header" href="#encoding-and-decoding-arena-allocated-types">Encoding and Decoding arena allocated types</a></h2> |
| <p>rustc has a lot of <a href="memory.html">arena allocated types</a>. |
| Deserializing these types isn't possible without access to the arena that they need to be allocated on. |
| The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.TyDecoder.html"><code>TyDecoder</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.TyEncoder.html"><code>TyEncoder</code></a> traits are subtraits of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_serialize/trait.Decoder.html"><code>Decoder</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_serialize/trait.Encoder.html"><code>Encoder</code></a> that allow access to a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html"><code>TyCtxt</code></a>.</p> |
| <p>Types which contain <code>arena</code> allocated types can then bound the type parameter of their |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_serialize/trait.Encodable.html"><code>Encodable</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_serialize/trait.Decodable.html"><code>Decodable</code></a> implementations with these traits. |
| For example</p> |
| <pre><code class="language-rust ignore">impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for MyStruct<'tcx> { |
| /* ... */ |
| }</code></pre> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_macros/derive.TyEncodable.html"><code>TyEncodable</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_macros/derive.TyDecodable.html"><code>TyDecodable</code></a> <a href="serialization.html#derive-macros">derive macros</a> will expand to such |
| an implementation.</p> |
| <p>Decoding the actual <code>arena</code> allocated type is harder, because some of the |
| implementations can't be written due to the <a href="https://doc.rust-lang.org/reference/items/implementations.html#orphan-rules">orphan rules</a>. To work around this, |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.RefDecodable.html"><code>RefDecodable</code></a> trait is defined in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/index.html"><code>rustc_middle</code></a>. This can then be |
| implemented for any type. The <code>TyDecodable</code> macro will call <code>RefDecodable</code> to |
| decode references, but various generic code needs types to actually be |
| <code>Decodable</code> with a specific decoder.</p> |
| <p>For interned types instead of manually implementing <code>RefDecodable</code>, using a new |
| type wrapper, like <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/predicate/struct.Predicate.html"><code>ty::Predicate</code></a> and manually implementing <code>Encodable</code> and |
| <code>Decodable</code> may be simpler.</p> |
| <h2 id="derive-macros"><a class="header" href="#derive-macros">Derive macros</a></h2> |
| <p>The <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_macros"><code>rustc_macros</code></a> crate defines various derives to help implement <code>Decodable</code> |
| and <code>Encodable</code>.</p> |
| <ul> |
| <li>The <code>Encodable</code> and <code>Decodable</code> macros generate implementations that apply to |
| all <code>Encoders</code> and <code>Decoders</code>. These should be used in crates that don't |
| depend on <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/index.html"><code>rustc_middle</code></a>, or that have to be serialized by a type that does |
| not implement <code>TyEncoder</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_macros/derive.MetadataEncodable.html"><code>MetadataEncodable</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_macros/derive.MetadataDecodable.html"><code>MetadataDecodable</code></a> generate implementations that |
| only allow decoding by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/encoder/struct.EncodeContext.html"><code>rustc_metadata::rmeta::encoder::EncodeContext</code></a> and |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/decoder/struct.DecodeContext.html"><code>rustc_metadata::rmeta::decoder::DecodeContext</code></a>. These are used for types |
| that contain <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/index.html"><code>rustc_metadata::rmeta::</code></a><code>Lazy*</code>.</li> |
| <li><code>TyEncodable</code> and <code>TyDecodable</code> generate implementation that apply to any |
| <code>TyEncoder</code> or <code>TyDecoder</code>. These should be used for types that are only |
| serialized in crate metadata and/or the incremental cache, which is most |
| serializable types in <code>rustc_middle</code>.</li> |
| </ul> |
| <h2 id="shorthands"><a class="header" href="#shorthands">Shorthands</a></h2> |
| <p><code>Ty</code> can be deeply recursive, if each <code>Ty</code> was encoded naively then crate |
| metadata would be very large. To handle this, each <code>TyEncoder</code> has a cache of |
| locations in its output where it has serialized types. If a type being encoded |
| is in the cache, then instead of serializing the type as usual, the byte offset |
| within the file being written is encoded instead. A similar scheme is used for |
| <code>ty::Predicate</code>.</p> |
| <h2 id="lazyvaluet"><a class="header" href="#lazyvaluet"><code>LazyValue<T></code></a></h2> |
| <p>Crate metadata is initially loaded before the <code>TyCtxt<'tcx></code> is created, so |
| some deserialization needs to be deferred from the initial loading of metadata. |
| The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/struct.LazyValue.html"><code>LazyValue<T></code></a> type wraps the (relative) offset in the crate metadata |
| where a <code>T</code> has been serialized. There are also some variants, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/struct.LazyValue.html"><code>LazyArray<T></code></a> |
| and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/struct.LazyValue.html"><code>LazyTable<I, T></code></a>.</p> |
| <p>The <code>LazyArray<[T]></code> and <code>LazyTable<I, T></code> types provide some functionality over |
| <code>Lazy<Vec<T>></code> and <code>Lazy<HashMap<I, T>></code>:</p> |
| <ul> |
| <li>It's possible to encode a <code>LazyArray<T></code> directly from an <code>Iterator</code>, without |
| first collecting into a <code>Vec<T></code>.</li> |
| <li>Indexing into a <code>LazyTable<I, T></code> does not require decoding entries other |
| than the one being read.</li> |
| </ul> |
| <p><strong>note</strong>: <code>LazyValue<T></code> does not cache its value after being deserialized the |
| first time. Instead the query system itself is the main way of caching these |
| results.</p> |
| <h2 id="specialization"><a class="header" href="#specialization">Specialization</a></h2> |
| <p>A few types, most notably <code>DefId</code>, need to have different implementations for |
| different <code>Encoder</code>s. This is currently handled by ad-hoc specializations, for |
| example: <code>DefId</code> has a <code>default</code> implementation of <code>Encodable<E></code> and a |
| specialized one for <code>Encodable<CacheEncoder></code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="parallel-compilation"><a class="header" href="#parallel-compilation">Parallel compilation</a></h1> |
| <div class="warning"> |
| As of <!-- date-check --> November 2024, |
| the parallel front-end is undergoing significant changes, |
| so this page contains quite a bit of outdated information. |
| <p>Tracking issue: <a href="https://github.com/rust-lang/rust/issues/113349">https://github.com/rust-lang/rust/issues/113349</a></p> |
| </div> |
| <p>As of <!-- date-check --> November 2024, most of the rust compiler is now |
| parallelized.</p> |
| <ul> |
| <li>The codegen part is executed concurrently by default. You can use the <code>-C codegen-units=n</code> option to control the number of concurrent tasks.</li> |
| <li>The parts after HIR lowering to codegen such as type checking, borrowing |
| checking, and mir optimization are parallelized in the nightly version. |
| Currently, they are executed in serial by default, and parallelization is |
| manually enabled by the user using the <code>-Z threads = n</code> option.</li> |
| <li>Other parts, such as lexical parsing, HIR lowering, and macro expansion, are |
| still executed in serial mode.</li> |
| </ul> |
| <div class="warning"> |
| The following sections are kept for now but are quite outdated. |
| </div> |
| <hr /> |
| <h2 id="code-generation-1"><a class="header" href="#code-generation-1">Code generation</a></h2> |
| <p>During monomorphization the compiler splits up all the code to |
| be generated into smaller chunks called <em>codegen units</em>. These are then generated by |
| independent instances of LLVM running in parallel. At the end, the linker |
| is run to combine all the codegen units together into one binary. This process |
| occurs in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/index.html"><code>rustc_codegen_ssa::base</code></a> module.</p> |
| <h2 id="data-structures"><a class="header" href="#data-structures">Data structures</a></h2> |
| <p>The underlying thread-safe data-structures used in the parallel compiler |
| can be found in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/sync/index.html"><code>rustc_data_structures::sync</code></a> module. These data structures |
| are implemented differently depending on whether <code>parallel-compiler</code> is true.</p> |
| <div class="table-wrapper"><table><thead><tr><th>data structure</th><th>parallel</th><th>non-parallel</th></tr></thead><tbody> |
| <tr><td>Lock<T></td><td>(parking_lot::Mutex<T>)</td><td>(std::cell::RefCell)</td></tr> |
| <tr><td>RwLock<T></td><td>(parking_lot::RwLock<T>)</td><td>(std::cell::RefCell)</td></tr> |
| <tr><td>MTLock<T></td><td>(Lock<T>)</td><td>(T)</td></tr> |
| <tr><td>ReadGuard</td><td>parking_lot::RwLockReadGuard</td><td>std::cell::Ref</td></tr> |
| <tr><td>MappedReadGuard</td><td>parking_lot::MappedRwLockReadGuard</td><td>std::cell::Ref</td></tr> |
| <tr><td>WriteGuard</td><td>parking_lot::RwLockWriteGuard</td><td>std::cell::RefMut</td></tr> |
| <tr><td>MappedWriteGuard</td><td>parking_lot::MappedRwLockWriteGuard</td><td>std::cell::RefMut</td></tr> |
| <tr><td>LockGuard</td><td>parking_lot::MutexGuard</td><td>std::cell::RefMut</td></tr> |
| </tbody></table> |
| </div> |
| <ul> |
| <li> |
| <p>These thread-safe data structures are interspersed during compilation which |
| can cause lock contention resulting in degraded performance as the number of |
| threads increases beyond 4. So we audit the use of these data structures |
| which leads to either a refactoring so as to reduce the use of shared state, |
| or the authoring of persistent documentation covering the specific of the |
| invariants, the atomicity, and the lock orderings.</p> |
| </li> |
| <li> |
| <p>On the other hand, we still need to figure out what other invariants |
| during compilation might not hold in parallel compilation.</p> |
| </li> |
| </ul> |
| <h3 id="workerlocal"><a class="header" href="#workerlocal">WorkerLocal</a></h3> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/sync/worker_local/struct.WorkerLocal.html"><code>WorkerLocal</code></a> is a special data structure implemented for parallel compilers. It |
| holds worker-locals values for each thread in a thread pool. You can only |
| access the worker local value through the <code>Deref</code> <code>impl</code> on the thread pool it |
| was constructed on. It panics otherwise.</p> |
| <p><code>WorkerLocal</code> is used to implement the <code>Arena</code> allocator in the parallel |
| environment, which is critical in parallel queries. Its implementation is |
| located in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/sync/worker_local/index.html"><code>rustc_data_structures::sync::worker_local</code></a> module. However, |
| in the non-parallel compiler, it is implemented as <code>(OneThread<T>)</code>, whose <code>T</code> |
| can be accessed directly through <code>Deref::deref</code>.</p> |
| <h2 id="parallel-iterator"><a class="header" href="#parallel-iterator">Parallel iterator</a></h2> |
| <p>The parallel iterators provided by the <a href="https://crates.io/crates/rayon"><code>rayon</code></a> crate are easy ways to |
| implement parallelism. In the current implementation of the parallel compiler |
| we use a custom <a href="https://github.com/rust-lang/rustc-rayon">fork</a> of <code>rayon</code> to run tasks in parallel.</p> |
| <p>Some iterator functions are implemented to run loops in parallel |
| when <code>parallel-compiler</code> is true.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Function(Omit <code>Send</code> and <code>Sync</code>)</th><th>Introduction</th><th>Owning Module</th></tr></thead><tbody> |
| <tr><td><strong>par_iter</strong><T: IntoParallelIterator>(t: T) -> T::Iter</td><td>generate a parallel iterator</td><td>rustc_data_structure::sync</td></tr> |
| <tr><td><strong>par_for_each_in</strong><T: IntoParallelIterator>(t: T, for_each: impl Fn(T::Item))</td><td>generate a parallel iterator and run <code>for_each</code> on each element</td><td>rustc_data_structure::sync</td></tr> |
| <tr><td><strong>Map::par_body_owners</strong>(self, f: impl Fn(LocalDefId))</td><td>run <code>f</code> on all hir owners in the crate</td><td>rustc_middle::hir::map</td></tr> |
| <tr><td><strong>Map::par_for_each_module</strong>(self, f: impl Fn(LocalDefId))</td><td>run <code>f</code> on all modules and sub modules in the crate</td><td>rustc_middle::hir::map</td></tr> |
| <tr><td><strong>ModuleItems::par_items</strong>(&self, f: impl Fn(ItemId))</td><td>run <code>f</code> on all items in the module</td><td>rustc_middle::hir</td></tr> |
| <tr><td><strong>ModuleItems::par_trait_items</strong>(&self, f: impl Fn(TraitItemId))</td><td>run <code>f</code> on all trait items in the module</td><td>rustc_middle::hir</td></tr> |
| <tr><td><strong>ModuleItems::par_impl_items</strong>(&self, f: impl Fn(ImplItemId))</td><td>run <code>f</code> on all impl items in the module</td><td>rustc_middle::hir</td></tr> |
| <tr><td><strong>ModuleItems::par_foreign_items</strong>(&self, f: impl Fn(ForeignItemId))</td><td>run <code>f</code> on all foreign items in the module</td><td>rustc_middle::hir</td></tr> |
| </tbody></table> |
| </div> |
| <p>There are a lot of loops in the compiler which can possibly be parallelized |
| using these functions. As of <!-- date-check--> August 2022, scenarios where |
| the parallel iterator function has been used are as follows:</p> |
| <div class="table-wrapper"><table><thead><tr><th>caller</th><th>scenario</th><th>callee</th></tr></thead><tbody> |
| <tr><td>rustc_metadata::rmeta::encoder::prefetch_mir</td><td>Prefetch queries which will be needed later by metadata encoding</td><td>par_iter</td></tr> |
| <tr><td>rustc_monomorphize::collector::collect_crate_mono_items</td><td>Collect monomorphized items reachable from non-generic items</td><td>par_for_each_in</td></tr> |
| <tr><td>rustc_interface::passes::analysis</td><td>Check the validity of the match statements</td><td>Map::par_body_owners</td></tr> |
| <tr><td>rustc_interface::passes::analysis</td><td>MIR borrow check</td><td>Map::par_body_owners</td></tr> |
| <tr><td>rustc_typeck::check::typeck_item_bodies</td><td>Type check</td><td>Map::par_body_owners</td></tr> |
| <tr><td>rustc_interface::passes::hir_id_validator::check_crate</td><td>Check the validity of hir</td><td>Map::par_for_each_module</td></tr> |
| <tr><td>rustc_interface::passes::analysis</td><td>Check the validity of loops body, attributes, naked functions, unstable abi, const bodys</td><td>Map::par_for_each_module</td></tr> |
| <tr><td>rustc_interface::passes::analysis</td><td>Liveness and intrinsic checking of MIR</td><td>Map::par_for_each_module</td></tr> |
| <tr><td>rustc_interface::passes::analysis</td><td>Deathness checking</td><td>Map::par_for_each_module</td></tr> |
| <tr><td>rustc_interface::passes::analysis</td><td>Privacy checking</td><td>Map::par_for_each_module</td></tr> |
| <tr><td>rustc_lint::late::check_crate</td><td>Run per-module lints</td><td>Map::par_for_each_module</td></tr> |
| <tr><td>rustc_typeck::check_crate</td><td>Well-formedness checking</td><td>Map::par_for_each_module</td></tr> |
| </tbody></table> |
| </div> |
| <p>There are still many loops that have the potential to use parallel iterators.</p> |
| <h2 id="query-system"><a class="header" href="#query-system">Query system</a></h2> |
| <p>The query model has some properties that make it actually feasible to evaluate |
| multiple queries in parallel without too much effort:</p> |
| <ul> |
| <li>All data a query provider can access is via the query context, so |
| the query context can take care of synchronizing access.</li> |
| <li>Query results are required to be immutable so they can safely be used by |
| different threads concurrently.</li> |
| </ul> |
| <p>When a query <code>foo</code> is evaluated, the cache table for <code>foo</code> is locked.</p> |
| <ul> |
| <li>If there already is a result, we can clone it, release the lock and |
| we are done.</li> |
| <li>If there is no cache entry and no other active query invocation computing the |
| same result, we mark the key as being "in progress", release the lock and |
| start evaluating.</li> |
| <li>If there <em>is</em> another query invocation for the same key in progress, we |
| release the lock, and just block the thread until the other invocation has |
| computed the result we are waiting for. <strong>Cycle error detection</strong> in the parallel |
| compiler requires more complex logic than in single-threaded mode. When |
| worker threads in parallel queries stop making progress due to interdependence, |
| the compiler uses an extra thread <em>(named deadlock handler)</em> to detect, remove and |
| report the cycle error.</li> |
| </ul> |
| <p>The parallel query feature still has implementation to do, most of which is |
| related to the previous <code>Data Structures</code> and <code>Parallel Iterators</code>. See <a href="https://github.com/rust-lang/rust/issues/48685">this |
| open feature tracking issue</a>.</p> |
| <h2 id="rustdoc-2"><a class="header" href="#rustdoc-2">Rustdoc</a></h2> |
| <p>As of <!-- date-check--> November 2022, there are still a number of steps to |
| complete before <code>rustdoc</code> rendering can be made parallel (see a open discussion |
| of <a href="https://github.com/rust-lang/rust/issues/82741">parallel <code>rustdoc</code></a>).</p> |
| <h2 id="resources-1"><a class="header" href="#resources-1">Resources</a></h2> |
| <p>Here are some resources that can be used to learn more:</p> |
| <ul> |
| <li><a href="https://internals.rust-lang.org/t/help-test-parallel-rustc/11503">This IRLO thread by alexchricton about performance</a></li> |
| <li><a href="https://internals.rust-lang.org/t/parallelizing-rustc-using-rayon/6606">This IRLO thread by Zoxc, one of the pioneers of the effort</a></li> |
| <li><a href="https://github.com/nikomatsakis/rustc-parallelization/blob/master/interior-mutability-list.md">This list of interior mutability in the compiler by nikomatsakis</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rustdoc-internals"><a class="header" href="#rustdoc-internals">Rustdoc internals</a></h1> |
| <p>This page describes <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/rustdoc"><code>rustdoc</code></a>'s passes and modes. |
| For an overview of <code>rustdoc</code>, see the <a href="./rustdoc.html">"Rustdoc overview" chapter</a>.</p> |
| <h2 id="from-crate-to-clean"><a class="header" href="#from-crate-to-clean">From crate to clean</a></h2> |
| <p>In <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/core.rs"><code>core.rs</code></a> are two central items: the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/core/struct.DocContext.html"><code>rustdoc::core::DocContext</code></a> |
| <code>struct</code>, and the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/core/fn.run_global_ctxt.html"><code>rustdoc::core::run_global_ctxt</code></a> function. |
| The latter is where <code>rustdoc</code> calls out to <code>rustc</code> to compile a crate to the point where |
| <code>rustdoc</code> can take over. |
| The former is a state container used when crawling through a crate to gather its documentation.</p> |
| <p>The main process of crate crawling is done in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/clean/mod.rs"><code>clean/mod.rs</code></a> through several |
| functions with names that start with <code>clean_</code>. |
| Each function accepts an <code>hir</code> |
| or <code>ty</code> data structure, and outputs a <code>clean</code> structure used by <code>rustdoc</code>. |
| For example, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/src/rustdoc/clean/mod.rs.html#256-267">this function for converting lifetimes</a>:</p> |
| <pre><code class="language-rust ignore">fn clean_lifetime<'tcx>(lifetime: &hir::Lifetime, cx: &mut DocContext<'tcx>) -> Lifetime { |
| if let Some( |
| rbv::ResolvedArg::EarlyBound(did) |
| | rbv::ResolvedArg::LateBound(_, _, did) |
| | rbv::ResolvedArg::Free(_, did), |
| ) = cx.tcx.named_bound_var(lifetime.hir_id) |
| && let Some(lt) = cx.args.get(&did).and_then(|arg| arg.as_lt()) |
| { |
| return lt.clone(); |
| } |
| Lifetime(lifetime.ident.name) |
| }</code></pre> |
| <p>Also, <code>clean/mod.rs</code> defines the types for the "cleaned" <a href="./ast-validation.html">Abstract Syntax Tree |
| (<code>AST</code>)</a> used later to render documentation pages. |
| Each usually accompanies a |
| <code>clean_*</code> function that takes some <a href="./ast-validation.html"><code>AST</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/index.html">High-Level Intermediate |
| Representation (<code>HIR</code>)</a> type from <code>rustc</code> and converts it into the appropriate "cleaned" type. |
| "Big" items like modules or associated items may |
| have some extra processing in its <code>clean</code> function, but for the most part these |
| <code>impl</code>s are straightforward conversions. |
| The "entry point" to this module is |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/clean/utils/fn.krate.html#"><code>clean::utils::krate</code></a>, which is called by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/core/fn.run_global_ctxt.html"><code>run_global_ctxt</code></a>.</p> |
| <p>The first step in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/src/rustdoc/clean/utils.rs.html#31-77"><code>clean::utils::krate</code></a> is to invoke |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/visit_ast/struct.RustdocVisitor.html"><code>visit_ast::RustdocVisitor</code></a> to process the module tree into an intermediate <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/visit_ast/struct.Module.html"><code>visit_ast::Module</code></a>. |
| This is the step that actually crawls the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Crate.html"><code>rustc_hir::Crate</code></a>, normalizing various aspects of name resolution, such as:</p> |
| <ul> |
| <li>handling <code>#[doc(inline)]</code> and <code>#[doc(no_inline)]</code></li> |
| <li>handling import globs and cycles, so there are no duplicates or infinite |
| directory trees</li> |
| <li>inlining public <code>use</code> exports of private items, or showing a "Reexport" |
| line in the module page</li> |
| <li>inlining items with <code>#[doc(hidden)]</code> if the base item is hidden but the</li> |
| <li>showing <code>#[macro_export]</code>-ed macros at the crate root, regardless of whether |
| they're defined as a reexport or not</li> |
| </ul> |
| <p>After this step, <code>clean::krate</code> invokes <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/clean/fn.clean_doc_module.html"><code>clean_doc_module</code></a>, which actually |
| converts the <code>HIR</code> items to the cleaned <a href="./ast-validation.html"><code>AST</code></a>. |
| This is also the step where cross-crate inlining is performed, |
| which requires converting <code>rustc_middle</code> data structures into the cleaned <a href="./ast-validation.html"><code>AST</code></a>.</p> |
| <p>The other major thing that happens in <code>clean/mod.rs</code> is the collection of doc |
| comments and <code>#[doc=""]</code> attributes into a separate field of the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/clean/types/struct.Attributes.html"><code>Attributes</code></a> |
| <code>struct</code>, present on anything that gets hand-written documentation. |
| This makes it easier to collect this documentation later in the process.</p> |
| <p>The primary output of this process is a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/clean/types/struct.Crate.html"><code>clean::types::Crate</code></a> with a tree of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/clean/types/struct.Item.html"><code>Item</code></a>s |
| which describe the publicly-documentable items in the target crate.</p> |
| <h3 id="passes-anything-but-a-gas-station-or-hot-potato"><a class="header" href="#passes-anything-but-a-gas-station-or-hot-potato">Passes Anything But a Gas Station (or: <a href="https://www.youtube.com/watch?v=WNFBIt5HxdY">Hot Potato</a>)</a></h3> |
| <p>Before moving on to the next major step, a few important "passes" occur over |
| the cleaned <a href="./ast-validation.html"><code>AST</code></a>. |
| Several of these passes are <code>lint</code>s and reports, but some of them mutate or generate new items.</p> |
| <p>These are all implemented in the <a href="https://github.com/rust-lang/rust/tree/HEAD/src/librustdoc/passes"><code>librustdoc/passes</code></a> directory, one file per pass. |
| By default, all of these passes are run on a crate, but the ones |
| regarding dropping private/hidden items can be bypassed by passing |
| <code>--document-private-items</code> to <code>rustdoc</code>. |
| Note that, unlike the previous set of <a href="./ast-validation.html"><code>AST</code></a> |
| transformations, the passes are run on the <em>cleaned</em> crate.</p> |
| <p>Here is the list of passes as of <!-- date-check --> March 2023:</p> |
| <ul> |
| <li> |
| <p><code>calculate-doc-coverage</code> calculates information used for the <code>--show-coverage</code> |
| flag.</p> |
| </li> |
| <li> |
| <p><code>check-doc-test-visibility</code> runs <code>doctest</code> visibility–related <code>lint</code>s. This pass |
| runs before <code>strip-private</code>, which is why it needs to be separate from <code>run-lints</code>.</p> |
| </li> |
| <li> |
| <p><code>collect-intra-doc-links</code> resolves <a href="https://doc.rust-lang.org/nightly/rustdoc/write-documentation/linking-to-items-by-name.html">intra-doc links</a>.</p> |
| </li> |
| <li> |
| <p><code>collect-trait-impls</code> collects <code>trait</code> <code>impl</code>s for each item in the crate. For |
| example, if we define a <code>struct</code> that implements a <code>trait</code>, this pass will note |
| that the <code>struct</code> implements that <code>trait</code>.</p> |
| </li> |
| <li> |
| <p><code>propagate-doc-cfg</code> propagates <code>#[doc(cfg(...))]</code> to child items.</p> |
| </li> |
| <li> |
| <p><code>run-lints</code> runs some of <code>rustdoc</code>'s <code>lint</code>s, defined in <code>passes/lint</code>. This is |
| the last pass to run.</p> |
| <ul> |
| <li> |
| <p><code>bare_urls</code> detects links that are not linkified, e.g., in Markdown such as |
| <code>Go to https://example.com/.</code> It suggests wrapping the link with angle brackets: |
| <code>Go to <https://example.com/>.</code> to linkify it. |
| This is the code behind the <!-- date-check: may 2022 --> <code>rustdoc::bare_urls</code> <code>lint</code>.</p> |
| </li> |
| <li> |
| <p><code>check_code_block_syntax</code> validates syntax inside Rust code blocks |
| (<code>```rust</code>)</p> |
| </li> |
| <li> |
| <p><code>html_tags</code> detects invalid <code>HTML</code> (like an unclosed <code><span></code>) |
| in doc comments.</p> |
| </li> |
| </ul> |
| </li> |
| <li> |
| <p><code>strip-hidden</code> and <code>strip-private</code> strip all <code>doc(hidden)</code> and private items |
| from the output. |
| <code>strip-private</code> implies <code>strip-priv-imports</code>. |
| Basically, the goal is to remove items that are not relevant for public documentation. |
| This pass is skipped when <code>--document-hidden-items</code> is passed.</p> |
| </li> |
| <li> |
| <p><code>strip-priv-imports</code> strips all private import statements (<code>use</code>, <code>extern crate</code>) from a crate. |
| This is necessary because <code>rustdoc</code> will handle <em>public</em> |
| imports by either inlining the item's documentation to the module or creating |
| a "Reexports" section with the import in it. |
| The pass ensures that all of these imports are actually relevant to documentation. |
| It is technically only run when <code>--document-private-items</code> is passed, but <code>strip-private</code> |
| accomplishes the same thing.</p> |
| </li> |
| <li> |
| <p><code>strip-private</code> strips all private items from a crate which cannot be seen |
| externally. |
| This pass is skipped when <code>--document-private-items</code> is passed.</p> |
| </li> |
| </ul> |
| <p>There is also a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/passes/stripper/index.html"><code>stripper</code></a> module in <code>librustdoc/passes</code>, but it is a |
| collection of utility functions for the <code>strip-*</code> passes and is not a pass itself.</p> |
| <h2 id="from-clean-to-html"><a class="header" href="#from-clean-to-html">From clean to HTML</a></h2> |
| <p>This is where the "second phase" in <code>rustdoc</code> begins. |
| This phase primarily lives |
| in the <a href="https://github.com/rust-lang/rust/tree/HEAD/src/librustdoc/formats"><code>librustdoc/formats</code></a> and <a href="https://github.com/rust-lang/rust/tree/HEAD/src/librustdoc/html"><code>librustdoc/html</code></a> folders, and it all starts with |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/formats/renderer/fn.run_format.html"><code>formats::renderer::run_format</code></a>. |
| This code is responsible for setting up a type that |
| <code>impl FormatRenderer</code>, which for <code>HTML</code> is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/html/render/context/struct.Context.html"><code>Context</code></a>.</p> |
| <p>This structure contains methods that get called by <code>run_format</code> to drive the |
| doc rendering, which includes:</p> |
| <ul> |
| <li><code>init</code> generates <code>static.files</code>, as well as search index and <code>src/</code></li> |
| <li><code>item</code> generates the item <code>HTML</code> files themselves</li> |
| <li><code>after_krate</code> generates other global resources like <code>all.html</code></li> |
| </ul> |
| <p>In <code>item</code>, the "page rendering" occurs, via a mixture of <a href="https://docs.rs/askama/latest/askama/">Askama</a> templates |
| and manual <code>write!()</code> calls, starting in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/html/layout.rs"><code>html/layout.rs</code></a>. |
| The parts that have not been converted to templates occur within a series of <code>std::fmt::Display</code> |
| implementations and functions that pass around a <code>&mut std::fmt::Formatter</code>.</p> |
| <p>The parts that actually generate <code>HTML</code> from the items and documentation start |
| with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/html/render/print_item/fn.print_item.html"><code>print_item</code></a> defined in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/html/render/print_item.rs"><code>html/render/print_item.rs</code></a>, which switches out |
| to one of several <code>item_*</code> functions based on kind of <code>Item</code> being rendered.</p> |
| <p>Depending on what kind of rendering code you're looking for, you'll probably |
| find it either in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/html/render/mod.rs"><code>html/render/mod.rs</code></a> for major items like "what sections |
| should I print for a <code>struct</code> page" or <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/html/format.rs"><code>html/format.rs</code></a> for smaller component |
| pieces like "how should I print a where clause as part of some other item".</p> |
| <p>Whenever <code>rustdoc</code> comes across an item that should print hand-written |
| documentation alongside, it calls out to <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/html/markdown.rs"><code>html/markdown.rs</code></a> which interfaces |
| with the Markdown parser. |
| This is exposed as a series of types that wrap a |
| string of Markdown, and implement <code>fmt::Display</code> to emit <code>HTML</code> text. |
| It takes special care to enable certain features like footnotes and tables and add |
| syntax highlighting to Rust code blocks (via <code>html/highlight.rs</code>) before |
| running the Markdown parser. |
| There's also a function <a href="https://doc.rust-lang.org/nightly/nightly-rustc/src/rustdoc/html/markdown.rs.html#749-818"><code>find_codes</code></a> which is |
| called by <code>find_testable_codes</code> that specifically scans for Rust code blocks so |
| the test-runner code can find all the <code>doctest</code>s in the crate.</p> |
| <h3 id="from-soup-to-nuts-or-an-unbroken-thread-stretches-from-those-first-cells-to-us"><a class="header" href="#from-soup-to-nuts-or-an-unbroken-thread-stretches-from-those-first-cells-to-us">From Soup to Nuts (or: <a href="https://www.youtube.com/watch?v=hOLAGYmUQV0">"An Unbroken Thread Stretches From Those First <code>Cell</code>s To Us"</a>)</a></h3> |
| <p>It's important to note that <code>rustdoc</code> can ask the compiler for type information |
| directly, even during <code>HTML</code> generation. |
| This <a href="https://github.com/rust-lang/rust/pull/80090">didn't used to be the case</a>, and |
| a lot of <code>rustdoc</code>'s architecture was designed around not doing that, but a |
| <code>TyCtxt</code> is now passed to <code>formats::renderer::run_format</code>, which is used to |
| run generation for both <code>HTML</code> and the (unstable as of <!-- date-check --> Nov 2025) JSON format.</p> |
| <p>This change has allowed other changes to remove data from the "clean" <a href="./ast-validation.html"><code>AST</code></a> |
| that can be easily derived from <code>TyCtxt</code> queries, and we'll usually accept |
| PRs that remove fields from "clean" (it's been soft-deprecated), but this |
| is complicated from two other constraints that <code>rustdoc</code> runs under:</p> |
| <ul> |
| <li>Docs can be generated for crates that don't actually pass type checking. |
| This is used for generating docs that cover mutually-exclusive platform |
| configurations, such as <code>libstd</code> having a single package of docs that |
| cover all supported operating systems. |
| This means <code>rustdoc</code> has to be able to generate docs from <code>HIR</code>.</li> |
| <li>Docs can inline across crates. Since crate metadata doesn't contain <code>HIR</code>, |
| it must be possible to generate inlined docs from the <code>rustc_middle</code> data.</li> |
| </ul> |
| <p>The "clean" <a href="./ast-validation.html"><code>AST</code></a> acts as a common output format for both input formats. |
| There is also some data in clean that doesn't correspond directly to <code>HIR</code>, such as |
| synthetic <code>impl</code>s for auto traits and blanket <code>impl</code>s generated by the <code>collect-trait-impls</code> pass.</p> |
| <p>Some additional data is stored in <code>html::render::context::{Context, SharedContext}</code>. |
| These two types serve as |
| ways to segregate <code>rustdoc</code>'s data for an eventual future with multithreaded doc |
| generation, as well as just keeping things organized:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/html/render/context/struct.Context.html"><code>Context</code></a> stores data used for generating the current page, such as its |
| path, a list of <code>HTML</code> IDs that have been used (to avoid duplicate <code>id=""</code>), |
| and the pointer to <code>SharedContext</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/html/render/context/struct.SharedContext.html"><code>SharedContext</code></a> stores data that does not vary by page, such as the <code>tcx</code> |
| pointer, and a list of all types.</li> |
| </ul> |
| <h2 id="other-tricks-up-its-sleeve"><a class="header" href="#other-tricks-up-its-sleeve">Other tricks up its sleeve</a></h2> |
| <p>All this describes the process for generating <code>HTML</code> documentation from a Rust |
| crate, but there are couple other major modes that <code>rustdoc</code> runs in. |
| It can also be run on a standalone Markdown file, or it can run <code>doctest</code>s on Rust code or |
| standalone Markdown files. |
| For the former, it shortcuts straight to |
| <code>html/markdown.rs</code>, optionally including a mode which inserts a Table of |
| Contents to the output <code>HTML</code>.</p> |
| <p>For the latter, <code>rustdoc</code> runs a similar partial-compilation to get relevant |
| documentation in <code>test.rs</code>, but instead of going through the full clean and |
| render process, it runs a much simpler crate walk to grab <em>just</em> the hand-written documentation. |
| Combined with the aforementioned |
| "<code>find_testable_code</code>" in <code>html/markdown.rs</code>, it builds up a collection of |
| tests to run before handing them off to the test runner. |
| One notable location in <code>test.rs</code> is the function <code>make_test</code>, which is where hand-written |
| <code>doctest</code>s get transformed into something that can be executed.</p> |
| <p>Some extra reading about <code>make_test</code> can be found |
| <a href="https://quietmisdreavus.net/code/2018/02/23/how-the-doctests-get-made/">here</a>.</p> |
| <h2 id="testing-locally"><a class="header" href="#testing-locally">Testing locally</a></h2> |
| <p>Some features of the generated <code>HTML</code> documentation might require local |
| storage to be used across pages, which doesn't work well without an <code>HTTP</code> server. |
| To test these features locally, you can run a local <code>HTTP</code> server, like this:</p> |
| <pre><code class="language-console">$ ./x doc library |
| # The documentation has been generated into `build/[YOUR ARCH]/doc`. |
| $ python3 -m http.server -d build/[YOUR ARCH]/doc |
| </code></pre> |
| <p>Now you can browse your documentation just like you would if it was hosted on the internet. |
| For example, the url for <code>std</code> will be <code>rust/std/</code>.</p> |
| <h2 id="see-also"><a class="header" href="#see-also">See also</a></h2> |
| <ul> |
| <li>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/"><code>rustdoc</code> api docs</a></li> |
| <li><a href="./rustdoc.html">An overview of <code>rustdoc</code></a></li> |
| <li><a href="https://doc.rust-lang.org/nightly/rustdoc/">The rustdoc user guide</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rustdoc-search"><a class="header" href="#rustdoc-search">Rustdoc search</a></h1> |
| <p>Rustdoc Search is two programs: <code>search_index.rs</code> |
| and <code>search.js</code>. The first generates a nasty JSON |
| file with a full list of items and function signatures |
| in the crates in the doc bundle, and the second reads |
| it, turns it into some in-memory structures, and |
| scans them linearly to search.</p> |
| <h2 id="search-index-format"><a class="header" href="#search-index-format">Search index format</a></h2> |
| <p><code>search.js</code> calls this Raw, because it turns it into |
| a more normal object tree after loading it. |
| For space savings, it's also written without newlines or spaces.</p> |
| <pre><code class="language-json">[ |
| [ "crate_name", { |
| // name |
| "n": ["function_name", "Data"], |
| // type |
| "t": "HF", |
| // parent module |
| "q": [[0, "crate_name"]], |
| // parent type |
| "i": [2, 0], |
| // type dictionary |
| "p": [[1, "i32"], [1, "str"], [5, "Data", 0]], |
| // function signature |
| "f": "{{gb}{d}}`", // [[3, 1], [2]] |
| // impl disambiguator |
| "b": [], |
| // deprecated flag |
| "c": "OjAAAAAAAAA=", // empty bitmap |
| // empty description flag |
| "e": "OjAAAAAAAAA=", // empty bitmap |
| // aliases |
| "a": [["get_name", 0]], |
| // description shards |
| "D": "g", // 3 |
| // inlined re-exports |
| "r": [], |
| }] |
| ] |
| </code></pre> |
| <p><a href="https://github.com/rust-lang/rust/blob/2f92f050e83bf3312ce4ba73c31fe843ad3cbc60/src/librustdoc/html/static/js/rustdoc.d.ts#L344-L390"><code>src/librustdoc/html/static/js/rustdoc.d.ts</code></a> |
| defines an actual schema in a TypeScript <code>type</code>.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Key</th><th>Name</th><th>Description</th></tr></thead><tbody> |
| <tr><td><code>n</code></td><td>Names</td><td>Item names</td></tr> |
| <tr><td><code>t</code></td><td>Item Type</td><td>One-char item type code</td></tr> |
| <tr><td><code>q</code></td><td>Parent module</td><td><code>Map<index, path></code></td></tr> |
| <tr><td><code>i</code></td><td>Parent type</td><td>list of indexes</td></tr> |
| <tr><td><code>f</code></td><td>Function signature</td><td><a href="rustdoc-internals/search.html#i-f-and-p">encoded</a></td></tr> |
| <tr><td><code>b</code></td><td>Impl disambiguator</td><td><code>Map<index, string></code></td></tr> |
| <tr><td><code>c</code></td><td>Deprecation flag</td><td><a href="rustdoc-internals/search.html#roaring-bitmaps">roaring bitmap</a></td></tr> |
| <tr><td><code>e</code></td><td>Description is empty</td><td><a href="rustdoc-internals/search.html#roaring-bitmaps">roaring bitmap</a></td></tr> |
| <tr><td><code>p</code></td><td>Type dictionary</td><td><code>[[item type, path]]</code></td></tr> |
| <tr><td><code>a</code></td><td>Alias</td><td><code>Map<string, index></code></td></tr> |
| <tr><td><code>D</code></td><td>description shards</td><td><a href="rustdoc-internals/search.html#how-descriptions-are-stored">encoded</a></td></tr> |
| </tbody></table> |
| </div> |
| <p>The above index defines a crate called <code>crate_name</code> |
| with a free function called <code>function_name</code> and a struct called <code>Data</code>, |
| with the type signature <code>Data, i32 -> str</code>, |
| and an alias, <code>get_name</code>, that equivalently refers to <code>function_name</code>.</p> |
| <p>The search index needs to fit the needs of the <code>rustdoc</code> compiler, |
| the <code>search.js</code> frontend, |
| and also be compact and fast to decode. |
| It makes a lot of compromises:</p> |
| <ul> |
| <li>The <code>rustdoc</code> compiler runs on one crate at a time, |
| so each crate has an essentially separate search index. |
| It <a href="https://github.com/rust-lang/rust/blob/79b710c13968a1a48d94431d024d2b1677940866/src/librustdoc/html/render/write_shared.rs#L151-L164">merges</a> them by having each crate on one line |
| and looking at the first quoted string.</li> |
| <li>Names in the search index are given |
| in their original case and with underscores. |
| When the search index is loaded, |
| <code>search.js</code> stores the original names for display, |
| but also folds them to lowercase and strips underscores for search. |
| You'll see them called <code>normalized</code>.</li> |
| <li>The <code>f</code> array stores types as offsets into the <code>p</code> array. |
| These types might actually be from another crate, |
| so <code>search.js</code> has to turn the numbers into names and then |
| back into numbers to deduplicate them if multiple crates in the |
| same index mention the same types.</li> |
| <li>It's a JSON file, but not designed to be human-readable. |
| Browsers already include an optimized JSON decoder, |
| so this saves on <code>search.js</code> code and performs better for small crates, |
| but instead of using objects like normal JSON formats do, |
| it tries to put data of the same type next to each other |
| so that the sliding window used by <a href="https://en.wikipedia.org/wiki/Deflate">DEFLATE</a> can find redundancies. |
| Where <code>search.js</code> does its own compression, |
| it's designed to save memory when the file is finally loaded, |
| not just size on disk or network transfer.</li> |
| </ul> |
| <h3 id="parallel-arrays-and-indexed-maps"><a class="header" href="#parallel-arrays-and-indexed-maps">Parallel arrays and indexed maps</a></h3> |
| <p>Abstractly, Rustdoc Search data is a table, stored in column-major form. |
| Most data in the index represents a set of parallel arrays |
| (the "columns") which refer to the same data if they're at the same position.</p> |
| <p>For example, |
| the above search index can be turned into this table:</p> |
| <div class="table-wrapper"><table><thead><tr><th></th><th>n</th><th>t</th><th><a href="rustdoc-internals/search.html#how-descriptions-are-stored">d</a></th><th>q</th><th>i</th><th>f</th><th>b</th><th>c</th></tr></thead><tbody> |
| <tr><td>0</td><td><code>crate_name</code></td><td><code>D</code></td><td>Documentation</td><td>NULL</td><td>0</td><td>NULL</td><td>NULL</td><td>0</td></tr> |
| <tr><td>1</td><td><code>function_name</code></td><td><code>H</code></td><td>This function gets the name of an integer with Data</td><td><code>crate_name</code></td><td>2</td><td><code>{{gb}{d}}</code></td><td>NULL</td><td>0</td></tr> |
| <tr><td>2</td><td><code>Data</code></td><td><code>F</code></td><td>The data struct</td><td><code>crate_name</code></td><td>0</td><td><code>`</code></td><td>NULL</td><td>0</td></tr> |
| </tbody></table> |
| </div> |
| <p>The crate row is implied in most columns, since its type is known (it's a crate), |
| it can't have a parent (crates form the root of the module tree), |
| its name is specified as the map key, |
| and function-specific data like the impl disambiguator can't apply either. |
| However, it can still have a description and it can still be deprecated. |
| The crate, therefore, has a primary key of <code>0</code>.</p> |
| <p>The above code doesn't use <code>c</code>, which holds deprecated indices, |
| or <code>b</code>, which maps indices to strings. |
| If <code>crate_name::function_name</code> used both, it might look like this.</p> |
| <pre><code class="language-json"> "b": [[0, "impl-Foo-for-Bar"]], |
| "c": "OjAAAAEAAAAAAAIAEAAAABUAbgZYCQ==", |
| </code></pre> |
| <p>This attaches a disambiguator to index 1 and marks it deprecated.</p> |
| <p>The advantage of this layout is that these APIs often have implicit structure |
| that DEFLATE can take advantage of, |
| but that rustdoc can't assume. |
| Like how names are usually CamelCase or snake_case, |
| but descriptions aren't. |
| It also makes it easier to use a sparse data for things like boolean flags.</p> |
| <p><code>q</code> is a Map from <em>the first applicable</em> ID to a parent module path. |
| This is a weird trick, but it makes more sense in pseudo-code:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>let mut parent_module = ""; |
| for (i, entry) in search_index.iter().enumerate() { |
| if q.contains(i) { |
| parent_module = q.get(i); |
| } |
| // ... do other stuff with `entry` ... |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This is valid because everything has a parent module |
| (even if it's just the crate itself), |
| and is easy to assemble because the rustdoc generator sorts by path |
| before serializing. |
| Doing this allows rustdoc to not only make the search index smaller, |
| but reuse the same string representing the parent path across multiple in-memory items.</p> |
| <h3 id="representing-sparse-columns"><a class="header" href="#representing-sparse-columns">Representing sparse columns</a></h3> |
| <h4 id="vlq-hex"><a class="header" href="#vlq-hex">VLQ Hex</a></h4> |
| <p>This format is, as far as I know, used nowhere other than rustdoc. |
| It follows this grammar:</p> |
| <pre><code class="language-ebnf">VLQHex = { VHItem | VHBackref } |
| VHItem = VHNumber | ( '{', {VHItem}, '}' ) |
| VHNumber = { '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' }, ( '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k ' | 'l' | 'm' | 'n' | 'o' ) |
| VHBackref = ( '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' ) |
| </code></pre> |
| <p>A VHNumber is a variable-length, self-terminating base16 number |
| (terminated because the last hexit is lowercase while all others are uppercase). |
| The sign bit is represented using <a href="https://en.wikipedia.org/wiki/Variable-length_quantity#Zigzag_encoding">zig-zag encoding</a>.</p> |
| <p>This alphabet is chosen because the characters can be turned into hexits by |
| masking off the last four bits of the ASCII encoding.</p> |
| <p>A major feature of this encoding, as with all of the "compression" done in rustdoc, |
| is that it can remain in its compressed format <em>even in memory at runtime</em>. |
| This is why <code>HBackref</code> is only used at the top level, |
| and why we don't just use <a href="https://en.wikipedia.org/wiki/Deflate">Flate</a> for everything: the decoder in search.js |
| will reuse the entire decoded object whenever a backref is seen, |
| saving decode work and memory.</p> |
| <h4 id="roaring-bitmaps"><a class="header" href="#roaring-bitmaps">Roaring Bitmaps</a></h4> |
| <p>Flag-style data, such as deprecation and empty descriptions, |
| are stored using the <a href="https://github.com/RoaringBitmap/RoaringFormatSpec">standard Roaring Bitmap serialization format with runs</a>. |
| The data is then base64 encoded when writing it.</p> |
| <p>As a brief overview: a roaring bitmap is a chunked array of bits, |
| described in <a href="https://arxiv.org/pdf/1603.06549.pdf">this paper</a>. |
| A chunk can either be a list of integers, a bitfield, or a list of runs. |
| In any case, the search engine has to base64 decode it, |
| and read the chunk index itself, |
| but the payload data stays as-is.</p> |
| <p>All roaring bitmaps in rustdoc currently store a flag for each item index. |
| The crate is item 0, all others start at 1.</p> |
| <h3 id="how-descriptions-are-stored"><a class="header" href="#how-descriptions-are-stored">How descriptions are stored</a></h3> |
| <p>The largest amount of data, |
| and the main thing Rustdoc Search deals with that isn't |
| actually used for searching, is descriptions. |
| In a SERP table, this is what appears on the rightmost column.</p> |
| <blockquote> |
| <div class="table-wrapper"><table><thead><tr><th>item type</th><th>item path</th><th><em><strong>description</strong></em> (this part)</th></tr></thead><tbody> |
| <tr><td>function</td><td>my_crate::my_function</td><td>This function gets the name of an integer with Data</td></tr> |
| </tbody></table> |
| </div></blockquote> |
| <p>When someone runs a search in rustdoc for the first time, their browser will |
| work through a "sandwich workload" of three steps:</p> |
| <ol> |
| <li>Download the search-index.js and search.js files (a network bottleneck).</li> |
| <li>Perform the actual search (a CPU and memory bandwidth bottleneck).</li> |
| <li>Download the description data (another network bottleneck).</li> |
| </ol> |
| <p>Reducing the amount of data downloaded here will almost always increase latency, |
| by delaying the decision of what to download behind other work and/or adding |
| data dependencies where something can't be downloaded without first downloading |
| something else. In this case, we can't start downloading descriptions until |
| after the search is done, because that's what allows it to decide <em>which</em> |
| descriptions to download (it needs to sort the results then truncate to 200).</p> |
| <p>To do this, two columns are stored in the search index, building on both |
| Roaring Bitmaps and on VLQ Hex.</p> |
| <ul> |
| <li><code>e</code> is an index of <strong>e</strong>mpty descriptions. It's a <a href="rustdoc-internals/search.html#roaring-bitmaps">roaring bitmap</a> of |
| each item (the crate itself is item 0, the rest start at 1).</li> |
| <li><code>D</code> is a shard list, stored in <a href="rustdoc-internals/search.html#vlq-hex">VLQ hex</a> as flat list of integers. |
| Each integer gives you the number of descriptions in the shard. |
| As the decoder walks the index, it checks if the description is empty. |
| if it's not, then it's in the "current" shard. When all items are |
| exhausted, it goes on to the next shard.</li> |
| </ul> |
| <p>Inside each shard is a newline-delimited list of descriptions, |
| wrapped in a JSONP-style function call.</p> |
| <h3 id="i-f-and-p"><a class="header" href="#i-f-and-p"><code>i</code>, <code>f</code>, and <code>p</code></a></h3> |
| <p><code>i</code> and <code>f</code> both index into <code>p</code>, the array of parent items.</p> |
| <p><code>i</code> is just a one-indexed number |
| (not zero-indexed because <code>0</code> is used for items that have no parent item). |
| It's different from <code>q</code> because <code>q</code> represents the parent <em>module or crate</em>, |
| which everything has, |
| while <code>i</code>/<code>q</code> are used for <em>type and trait-associated items</em> like methods.</p> |
| <p><code>f</code>, the function signatures, use a <a href="rustdoc-internals/search.html#vlq-hex">VLQ hex</a> tree. |
| A number is either a one-indexed reference into <code>p</code>, |
| a negative number representing a generic, |
| or zero for null.</p> |
| <p>(the internal object representation also uses negative numbers, |
| even after decoding, |
| to represent generics).</p> |
| <p>For example, <code>{{gb}{d}}</code> is equivalent to the json <code>[[3, 1], [2]]</code>. |
| Because of zigzag encoding, <code>`</code> is +0, <code>a</code> is -0 (which is not used), |
| <code>b</code> is +1, and <code>c</code> is -1.</p> |
| <h2 id="searching-by-name"><a class="header" href="#searching-by-name">Searching by name</a></h2> |
| <p>Searching by name works by looping through the search index |
| and running these functions on each:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/blob/79b710c13968a1a48d94431d024d2b1677940866/src/librustdoc/html/static/js/search.js#L137"><code>editDistance</code></a> is always used to determine a match |
| (unless quotes are specified, which would use simple equality instead). |
| It computes the number of swaps, inserts, and removes needed to turn |
| the query name into the entry name. |
| For example, <code>foo</code> has zero distance from itself, |
| but a distance of 1 from <code>ofo</code> (one swap) and <code>foob</code> (one insert). |
| It is checked against an heuristic threshold, and then, |
| if it is within that threshold, the distance is stored for ranking.</li> |
| <li><a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/indexOf"><code>String.prototype.indexOf</code></a> is always used to determine a match. |
| If it returns anything other than -1, the result is added, |
| even if <code>editDistance</code> exceeds its threshold, |
| and the index is stored for ranking.</li> |
| <li><a href="https://github.com/rust-lang/rust/blob/79b710c13968a1a48d94431d024d2b1677940866/src/librustdoc/html/static/js/search.js#L1814"><code>checkPath</code></a> is used if, and only if, a parent path is specified |
| in the query. For example, <code>vec</code> has no parent path, but <code>vec::vec</code> does. |
| Within checkPath, editDistance and indexOf are used, |
| and the path query has its own heuristic threshold, too. |
| If it's not within the threshold, the entry is rejected, |
| even if the first two pass. |
| If it's within the threshold, the path distance is stored |
| for ranking.</li> |
| <li><a href="https://github.com/rust-lang/rust/blob/79b710c13968a1a48d94431d024d2b1677940866/src/librustdoc/html/static/js/search.js#L1787"><code>checkType</code></a> is used only if there's a type filter, |
| like the struct in <code>struct:vec</code>. If it fails, |
| the entry is rejected.</li> |
| </ul> |
| <p>If all four criteria pass |
| (plus the crate filter, which isn't technically part of the query), |
| the results are sorted by <a href="https://github.com/rust-lang/rust/blob/79b710c13968a1a48d94431d024d2b1677940866/src/librustdoc/html/static/js/search.js#L1229"><code>sortResults</code></a>.</p> |
| <h2 id="searching-by-type"><a class="header" href="#searching-by-type">Searching by type</a></h2> |
| <p>Searching by type can be divided into two phases, |
| and the second phase has two sub-phases.</p> |
| <ul> |
| <li>Turn names in the query into numbers.</li> |
| <li>Loop over each entry in the search index: |
| <ul> |
| <li>Quick rejection using a bloom filter.</li> |
| <li>Slow rejection using a recursive type unification algorithm.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>In the names->numbers phase, if the query has only one name in it, |
| the editDistance function is used to find a near match if the exact match fails, |
| but if there's multiple items in the query, |
| non-matching items are treated as generics instead. |
| This means <code>hahsmap</code> will match hashmap on its own, but <code>hahsmap, u32</code> |
| is going to match the same things <code>T, u32</code> matches |
| (though rustdoc will detect this particular problem and warn about it).</p> |
| <p>Then, when actually looping over each item, |
| the bloom filter will probably reject entries that don't have every |
| type mentioned in the query. |
| For example, the bloom query allows a query of <code>i32 -> u32</code> to match |
| a function with the type <code>i32, u32 -> bool</code>, |
| but unification will reject it later.</p> |
| <p>The unification filter ensures that:</p> |
| <ul> |
| <li>Bag semantics are respected. If you query says <code>i32, i32</code>, |
| then the function has to mention <em>two</em> i32s, not just one.</li> |
| <li>Nesting semantics are respected. If your query says <code>vec<option></code>, |
| then <code>vec<option<i32>></code> is fine, but <code>option<vec<i32>></code> <em>is not</em> a match.</li> |
| <li>The division between return type and parameter is respected. |
| <code>i32 -> u32</code> and <code>u32 -> i32</code> are completely different.</li> |
| </ul> |
| <p>The bloom filter checks none of these things, |
| and, on top of that, can have false positives. |
| But it's fast and uses very little memory, so the bloom filter helps.</p> |
| <h2 id="re-exports"><a class="header" href="#re-exports">Re-exports</a></h2> |
| <p><a href="https://doc.rust-lang.org/nightly/rustdoc/write-documentation/re-exports.html">Re-export inlining</a> allows the same item to be found by multiple names. |
| Search supports this by giving the same item multiple entries and tracking a canonical path |
| for any items where that differs from the given path.</p> |
| <p>For example, this sample index has a single struct exported from two paths:</p> |
| <pre><code class="language-json">[ |
| [ "crate_name", { |
| "doc": "Documentation", |
| "n": ["Data", "Data"], |
| "t": "FF", |
| "d": ["The data struct", "The data struct"], |
| "q": [[0, "crate_name"], [1, "crate_name::submodule"]], |
| "i": [0, 0], |
| "p": [], |
| "f": "``", |
| "b": [], |
| "c": [], |
| "a": [], |
| "r": [[0, 1]], |
| }] |
| ] |
| </code></pre> |
| <p>The important part of this example is the <code>r</code> array, |
| which indicates that path entry 1 in the <code>q</code> array is |
| the canonical path for item 0. |
| That is, <code>crate_name::Data</code> has a canonical path of <code>crate_name::submodule::Data</code>.</p> |
| <p>This might sound like a strange design, since it has the duplicate data. |
| It's done that way because inlining can happen across crates, |
| which are compiled separately and might not all be present in the docs.</p> |
| <pre><code class="language-json">[ |
| [ "crate_name", ... ], |
| [ "crate_name_2", { "q": [[0, "crate_name::submodule"], [5, "core::option"]], ... }] |
| ] |
| </code></pre> |
| <p>In the above example, a canonical path actually comes from a dependency, |
| and another one comes from an inlined standard library item: |
| the canonical path isn't even in the index! |
| The canonical path might also be private. |
| In either case, it's never shown to the user, and is only used for deduplication.</p> |
| <p>Associated types, like methods, store them differently. |
| These types are connected with an entry in <code>p</code> (their "parent") |
| and each one has an optional third tuple element:</p> |
| <pre><code>"p": [[5, "Data", 0, 1]] |
| </code></pre> |
| <p>That's:</p> |
| <ul> |
| <li>5: It's a struct</li> |
| <li>"Data": Its name</li> |
| <li>0: Its display path, "crate_name"</li> |
| <li>1: Its canonical path, "crate_name::submodule"</li> |
| </ul> |
| <p>In both cases, the canonical path might not be public at all, |
| or it might be from another crate that isn't in the docs, |
| so it's never shown to the user, but is used for deduplication.</p> |
| <h2 id="testing-the-search-engine"><a class="header" href="#testing-the-search-engine">Testing the search engine</a></h2> |
| <p>While the generated UI is tested using <code>rustdoc-gui</code> tests, the |
| primary way the search engine is tested is the <code>rustdoc-js</code> and |
| <code>rustdoc-js-std</code> tests. They run in NodeJS.</p> |
| <p>A <code>rustdoc-js</code> test has a <code>.rs</code> and <code>.js</code> file, with the same name. |
| The <code>.rs</code> file specifies the hypothetical library crate to run |
| the searches on (make sure you mark anything you need to find as <code>pub</code>). |
| The <code>.js</code> file specifies the actual searches. |
| The <code>rustdoc-js-std</code> tests are the same, but don't require an <code>.rs</code> |
| file, since they use the standard library.</p> |
| <p>The <code>.js</code> file is like a module (except the loader takes care of |
| <code>exports</code> for you). It uses these variables:</p> |
| <div class="table-wrapper"><table><thead><tr><th>Name</th><th>Type</th><th>Description</th></tr></thead><tbody> |
| <tr><td><code>FILTER_CRATE</code></td><td><code>string</code></td><td>Only include results from the given crate. In the GUI, this is the "Results in <kbd>crate</kbd>" drop-down menu.</td></tr> |
| <tr><td><code>EXPECTED</code></td><td><code>[ResultsTable]|ResultsTable</code></td><td>List of tests to run, specifying what the hypothetical user types into the search box and sees in the tabs</td></tr> |
| <tr><td><code>PARSED</code></td><td><code>[ParsedQuery]|ParsedQuery</code></td><td>List of parser tests to run, without running an actual search</td></tr> |
| </tbody></table> |
| </div> |
| <p><code>FILTER_CRATE</code> can be left out (equivalent to searching "all crates"), but you |
| have to specify <code>EXPECTED</code> or <code>PARSED</code>.</p> |
| <p>By default, the test fails if any of the results specified in the test case are |
| not found after running the search, or if the results found after running the |
| search don't appear in the same order that they do in the test. |
| The actual search results may, however, include results that aren't in the test. |
| To override this, specify any of the following magic comments. |
| Put them on their own line, without indenting.</p> |
| <ul> |
| <li><code>// exact-check</code>: If search results appear that aren't part of the test case, |
| then fail.</li> |
| <li><code>// ignore-order</code>: Allow search results to appear in any order.</li> |
| <li><code>// should-fail</code>: Used to write negative tests.</li> |
| </ul> |
| <p>Standard library tests usually shouldn't specify <code>// exact-check</code>, since we |
| want the libs team to be able to add new items without causing unrelated |
| tests to fail, but standalone tests will use it more often.</p> |
| <p>The <code>ResultsTable</code> and <code>ParsedQuery</code> types are specified in |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/html/static/js/rustdoc.d.ts"><code>rustdoc.d.ts</code></a>.</p> |
| <p>For example, imagine we needed to fix a bug where a function named |
| <code>constructor</code> couldn't be found. To do this, write two files:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// tests/rustdoc-js/constructor_search.rs |
| // The test case needs to find this result. |
| pub fn constructor(_input: &str) -> i32 { 1 } |
| <span class="boring">}</span></code></pre></pre> |
| <pre><code class="language-js">// tests/rustdoc-js/constructor_search.js |
| // exact-check |
| // Since this test runs against its own crate, |
| // new items should not appear in the search results. |
| const EXPECTED = [ |
| // This first test targets name-based search. |
| { |
| query: "constructor", |
| others: [ |
| { path: "constructor_search", name: "constructor" }, |
| ], |
| in_args: [], |
| returned: [], |
| }, |
| // This test targets the second tab. |
| { |
| query: "str", |
| others: [], |
| in_args: [ |
| { path: "constructor_search", name: "constructor" }, |
| ], |
| returned: [], |
| }, |
| // This test targets the third tab. |
| { |
| query: "i32", |
| others: [], |
| in_args: [], |
| returned: [ |
| { path: "constructor_search", name: "constructor" }, |
| ], |
| }, |
| // This test targets advanced type-driven search. |
| { |
| query: "str -> i32", |
| others: [ |
| { path: "constructor_search", name: "constructor" }, |
| ], |
| in_args: [], |
| returned: [], |
| }, |
| ] |
| </code></pre> |
| <p>If the <a href="rustdoc-internals/../tests/compiletest.html#revisions"><code>//@ revisions</code></a> directive is used, the JS file will |
| have access to a variable called <code>REVISION</code>.</p> |
| <pre><code class="language-js">const EXPECTED = [ |
| // This first test targets name-based search. |
| { |
| query: "constructor", |
| others: REVISION === "has_constructor" ? |
| [ |
| { path: "constructor_search", name: "constructor" }, |
| ] : |
| [], |
| in_args: [], |
| returned: [], |
| }, |
| ]; |
| </code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-rustdoc-test-suite"><a class="header" href="#the-rustdoc-test-suite">The <code>rustdoc</code> test suite</a></h1> |
| <p>This page is about the test suite named <code>rustdoc</code> used to test the HTML output of <code>rustdoc</code>. |
| For other rustdoc-specific test suites, see <a href="rustdoc-internals/../tests/compiletest.html#rustdoc-test-suites">Rustdoc test suites</a>.</p> |
| <p>Each test file in this test suite is simply a Rust source file <code>file.rs</code> sprinkled with |
| so-called <em>directives</em> located inside normal Rust code comments. |
| These come in two flavors: <em>Compiletest</em> and <em>HtmlDocCk</em>.</p> |
| <p>To learn more about the former, read <a href="rustdoc-internals/../tests/directives.html">Compiletest directives</a>. |
| For the latter, continue reading.</p> |
| <p>Internally, <a href="rustdoc-internals/../tests/compiletest.html"><code>compiletest</code></a> invokes the supplementary checker script <a href="https://github.com/rust-lang/rust/blob/HEAD/src/etc/htmldocck.py"><code>htmldocck.py</code></a>.</p> |
| <h2 id="htmldocck-directives"><a class="header" href="#htmldocck-directives">HtmlDocCk Directives</a></h2> |
| <p>Directives to HtmlDocCk are assertions that place constraints on the generated HTML. |
| They look similar to those given to <code>compiletest</code> in that they take the form of <code>//@</code> comments |
| but ultimately, they are completely distinct and processed by different programs.</p> |
| <p><a href="https://en.wikipedia.org/wiki/XPath">XPath</a> is used to query parts of the HTML document tree.</p> |
| <p><strong>Introductory example</strong>:</p> |
| <pre><code class="language-rust ignore (illustrative)">//@ has file/type.Alias.html |
| //@ has - '//*[@class="rust item-decl"]//code' 'type Alias = Option<i32>;' |
| pub type Alias = Option<i32>;</code></pre> |
| <p>Here, we check that documentation generated for crate <code>file</code> contains a page for the |
| public type alias <code>Alias</code> where the code block that is found at the top contains the |
| expected rendering of the item. The <code>//*[@class="rust item-decl"]//code</code> is an XPath |
| expression.</p> |
| <p>Conventionally, you place these directives directly above the thing they are meant to test. |
| Technically speaking however, they don't need to be as HtmlDocCk only looks for the directives.</p> |
| <p>All directives take a <code>PATH</code> argument. |
| To avoid repetition, <code>-</code> can be passed to it to re-use the previous <code>PATH</code> argument. |
| Since the path contains the name of the crate, it is conventional to add a |
| <code>#![crate_name = "foo"]</code> attribute to the crate root to shorten the resulting path.</p> |
| <p>All arguments take the form of shell-style (single or double) quoted strings, |
| with the exception of <code>COUNT</code> and the special <code>-</code> form of <code>PATH</code>.</p> |
| <p>All directives (except <code>files</code>) can be <em>negated</em> by putting a <code>!</code> in front of their name. |
| Before you add negated directives, please read about <a href="rustdoc-internals/rustdoc-test-suite.html#caveats">their caveats</a>.</p> |
| <p>Similar to shell commands, |
| directives can extend across multiple lines if their last char is <code>\</code>. |
| In this case, the start of the next line should be <code>//</code>, with no <code>@</code>.</p> |
| <p>Similar to compiletest directives, besides a space you can also use a colon <code>:</code> to separate |
| the directive name and the arguments, however a space is preferred for HtmlDocCk directives.</p> |
| <p>Use the special string <code>{{channel}}</code> in XPaths, <code>PATTERN</code> arguments and <a href="rustdoc-internals/rustdoc-test-suite.html#snapshot">snapshot files</a> |
| if you'd like to refer to the URL <code>https://doc.rust-lang.org/CHANNEL</code> where <code>CHANNEL</code> refers to the |
| current release channel (e.g, <code>stable</code> or <code>nightly</code>).</p> |
| <p>Listed below are all possible directives:</p> |
| <h3 id="has"><a class="header" href="#has"><code>has</code></a></h3> |
| <blockquote> |
| <p>Usage 1: <code>//@ has PATH</code></p> |
| </blockquote> |
| <p>Check that the file given by <code>PATH</code> exists.</p> |
| <blockquote> |
| <p>Usage 2: <code>//@ has PATH XPATH PATTERN</code></p> |
| </blockquote> |
| <p>Checks that the text of each element / attribute / text selected by <code>XPATH</code> in the |
| whitespace-normalized<sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup> file given by <code>PATH</code> matches the |
| (also whitespace-normalized) string <code>PATTERN</code>.</p> |
| <p><strong>Tip</strong>: If you'd like to avoid whitespace normalization and/or if you'd like to match with a regex, |
| use <code>matches</code> instead.</p> |
| <h3 id="hasraw"><a class="header" href="#hasraw"><code>hasraw</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ hasraw PATH PATTERN</code></p> |
| </blockquote> |
| <p>Checks that the contents of the whitespace-normalized<sup class="footnote-reference" id="fr-1-2"><a href="#footnote-1">1</a></sup> file given by <code>PATH</code> |
| matches the (also whitespace-normalized) string <code>PATTERN</code>.</p> |
| <p><strong>Tip</strong>: If you'd like to avoid whitespace normalization and / or if you'd like to match with a |
| regex, use <code>matchesraw</code> instead.</p> |
| <h3 id="matches"><a class="header" href="#matches"><code>matches</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ matches PATH XPATH PATTERN</code></p> |
| </blockquote> |
| <p>Checks that the text of each element / attribute / text selected by <code>XPATH</code> in the |
| file given by <code>PATH</code> matches the Python-flavored<sup class="footnote-reference" id="fr-2-1"><a href="#footnote-2">2</a></sup> regex <code>PATTERN</code>.</p> |
| <h3 id="matchesraw"><a class="header" href="#matchesraw"><code>matchesraw</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ matchesraw PATH PATTERN</code></p> |
| </blockquote> |
| <p>Checks that the contents of the file given by <code>PATH</code> matches the |
| Python-flavored<sup class="footnote-reference" id="fr-2-2"><a href="#footnote-2">2</a></sup> regex <code>PATTERN</code>.</p> |
| <h3 id="count"><a class="header" href="#count"><code>count</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ count PATH XPATH COUNT</code></p> |
| </blockquote> |
| <p>Checks that there are exactly <code>COUNT</code> matches for <code>XPATH</code> within the file given by <code>PATH</code>.</p> |
| <h3 id="snapshot"><a class="header" href="#snapshot"><code>snapshot</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ snapshot NAME PATH XPATH</code></p> |
| </blockquote> |
| <p>Checks that the element / text selected by <code>XPATH</code> in the file given by <code>PATH</code> matches the |
| pre-recorded subtree or text (the "snapshot") in file <code>FILE_STEM.NAME.html</code> where <code>FILE_STEM</code> |
| is the file stem of the test file.</p> |
| <p>Pass the <code>--bless</code> option to <code>compiletest</code> to accept the current subtree/text as expected. |
| This will overwrite the aforementioned file (or create it if it doesn't exist). It will |
| automatically normalize the channel-dependent URL <code>https://doc.rust-lang.org/CHANNEL</code> to |
| the special string <code>{{channel}}</code>.</p> |
| <h3 id="has-dir"><a class="header" href="#has-dir"><code>has-dir</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ has-dir PATH</code></p> |
| </blockquote> |
| <p>Checks for the existence of the directory given by <code>PATH</code>.</p> |
| <h3 id="files"><a class="header" href="#files"><code>files</code></a></h3> |
| <blockquote> |
| <p>Usage: <code>//@ files PATH ENTRIES</code></p> |
| </blockquote> |
| <p>Checks that the directory given by <code>PATH</code> contains exactly <code>ENTRIES</code>. |
| <code>ENTRIES</code> is a Python-like list of strings inside a quoted string.</p> |
| <p><strong>Example</strong>: <code>//@ files "foo/bar" '["index.html", "sidebar-items.js"]'</code></p> |
| <h2 id="compiletest-directives-brief"><a class="header" href="#compiletest-directives-brief">Compiletest Directives (Brief)</a></h2> |
| <p>As mentioned in the introduction, you also have access to <a href="rustdoc-internals/../tests/directives.html">compiletest directives</a>. |
| Most importantly, they allow you to register auxiliary crates and |
| to pass flags to the <code>rustdoc</code> binary under test. |
| It's <em>strongly recommended</em> to read that chapter if you don't know anything about them yet.</p> |
| <p>Here are some details that are relevant to this test suite specifically:</p> |
| <ul> |
| <li>While you can use both <code>//@ compile-flags</code> and <code>//@ doc-flags</code> to pass flags to <code>rustdoc</code>, |
| prefer to user the latter to show intent. The former is meant for <code>rustc</code>.</li> |
| <li>Add <code>//@ build-aux-docs</code> to the test file that has auxiliary crates to not only compile the |
| auxiliaries with <code>rustc</code> but to also document them with <code>rustdoc</code>.</li> |
| </ul> |
| <h2 id="caveats"><a class="header" href="#caveats">Caveats</a></h2> |
| <p>Testing for the absence of an element or a piece of text is quite fragile and not very future proof.</p> |
| <p>It's not unusual that the <em>shape</em> of the generated HTML document tree changes from time to time. |
| This includes for example renamings of CSS classes.</p> |
| <p>Whenever that happens, <em>positive</em> checks will either continue to match the intended element / |
| attribute / text (if their XPath expression is general / loose enough) and |
| thus continue to test the correct thing or they won't in which case they would fail thereby |
| forcing the author of the change to look at them.</p> |
| <p>Compare that to <em>negative</em> checks (e.g., <code>//@ !has PATH XPATH PATTERN</code>) which won't fail if their |
| XPath expression "no longer" matches. The author who changed "the shape" thus won't get notified and |
| as a result someone else can unintentionally reintroduce <code>PATTERN</code> into the generated docs without |
| the original negative check failing.</p> |
| <p><strong>Note</strong>: Please avoid the use of <em>negated</em> checks!</p> |
| <p><strong>Tip</strong>: If you can't avoid it, please <strong>always</strong> pair it with an analogous positive check in the |
| immediate vicinity, so people changing "the shape" have a chance to notice and to update the |
| negated check!</p> |
| <h2 id="limitations"><a class="header" href="#limitations">Limitations</a></h2> |
| <p>HtmlDocCk uses the XPath implementation from the Python standard library. |
| This leads to several limitations:</p> |
| <ul> |
| <li>All <code>XPATH</code> arguments must start with <code>//</code> due to a flaw in the implementation.</li> |
| <li>Many XPath features (functions, axies, etc.) are not supported.</li> |
| <li>Only well-formed HTML can be parsed (hopefully rustdoc doesn't output mismatched tags).</li> |
| </ul> |
| <p>Furthmore, compiletest <a href="rustdoc-internals/../tests/compiletest.html#revisions">revisions</a> are not supported.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>Whitespace normalization means that all spans of consecutive whitespace are replaced with a single space. <a href="#fr-1-1">↩</a> <a href="#fr-1-2">↩2</a></p> |
| </li> |
| <li id="footnote-2"> |
| <p>They are Unicode aware (flag <code>UNICODE</code> is set), match case-sensitively and in single-line mode. <a href="#fr-2-1">↩</a> <a href="#fr-2-2">↩2</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="the-rustdoc-gui-test-suite"><a class="header" href="#the-rustdoc-gui-test-suite">The <code>rustdoc-gui</code> test suite</a></h1> |
| <blockquote> |
| <p><strong>FIXME</strong>: This section is a stub. Please help us flesh it out!</p> |
| </blockquote> |
| <p>This page is about the test suite named <code>rustdoc-gui</code> used to test the "GUI" of <code>rustdoc</code> (i.e., the HTML/JS/CSS as rendered in a browser). |
| For other rustdoc-specific test suites, see <a href="rustdoc-internals/../tests/compiletest.html#rustdoc-test-suites">Rustdoc test suites</a>.</p> |
| <p>These use a NodeJS-based tool called <a href="https://github.com/GuillaumeGomez/browser-UI-test/"><code>browser-UI-test</code></a> that uses <a href="https://pptr.dev/">puppeteer</a> to run tests in a headless browser and check rendering and interactivity. For information on how to write this form of test, see <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/rustdoc-gui/README.md"><code>tests/rustdoc-gui/README.md</code></a> as well as <a href="https://github.com/GuillaumeGomez/browser-UI-test/blob/main/goml-script.md">the description of the <code>.goml</code> format</a></p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-rustdoc-json-test-suite"><a class="header" href="#the-rustdoc-json-test-suite">The <code>rustdoc-json</code> test suite</a></h1> |
| <p>This page is specifically about the test suite named <code>rustdoc-json</code>, which tests rustdoc's <a href="https://doc.rust-lang.org/nightly/rustdoc/unstable-features.html#json-output">json output</a>. |
| For other test suites used for testing rustdoc, see <a href="rustdoc-internals/../tests/compiletest.html#rustdoc-test-suites">§Rustdoc test suites</a>.</p> |
| <p>Tests are run with compiletest, and have access to the usual set of <a href="rustdoc-internals/../tests/directives.html">directives</a>. |
| Frequenly used directives here are:</p> |
| <ul> |
| <li><a href="rustdoc-internals/../tests/compiletest.html#building-auxiliary-crates"><code>//@ aux-build</code></a> to have dependencies.</li> |
| <li><code>//@ edition: 2021</code> (or some other edition).</li> |
| <li><code>//@ compile-flags: --document-hidden-items</code> to enable <a href="https://doc.rust-lang.org/nightly/rustdoc/command-line-arguments.html#--document-private-items-show-items-that-are-not-public">document private items</a>.</li> |
| </ul> |
| <p>Each crate's json output is checked by 2 programs: <a href="rustdoc-internals/rustdoc-json-test-suite.html#jsondocck">jsondoclint</a> and <a href="rustdoc-internals/rustdoc-json-test-suite.html#jsondocck">jsondocck</a>.</p> |
| <h2 id="jsondoclint"><a class="header" href="#jsondoclint">jsondoclint</a></h2> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/jsondoclint">jsondoclint</a> checks that all <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types/struct.Id.html"><code>Id</code></a>s exist in the <code>index</code> (or <code>paths</code>). |
| This makes sure there are no dangling <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types/struct.Id.html"><code>Id</code></a>s.</p> |
| <!-- TODO: It does some more things too? |
| Also, talk about how it works |
| --> |
| <h2 id="jsondocck"><a class="header" href="#jsondocck">jsondocck</a></h2> |
| <p><a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/jsondocck">jsondocck</a> processes direcives given in comments, to assert that the values in the output are expected. |
| It's alot like <a href="rustdoc-internals/./rustdoc-test-suite.html">htmldocck</a> in that way.</p> |
| <p>It uses <a href="https://www.rfc-editor.org/rfc/rfc9535.html">JSONPath</a> as a query language, which takes a path, and returns a <em>list</em> of values that that path is said to match to.</p> |
| <h3 id="directives"><a class="header" href="#directives">Directives</a></h3> |
| <ul> |
| <li><code>//@ has <path></code>: Checks <code><path></code> exists, i.e. matches at least 1 value.</li> |
| <li><code>//@ !has <path></code>: Checks <code><path></code> doesn't exist, i.e. matches 0 values.</li> |
| <li><code>//@ has <path> <value></code>: Check <code><path></code> exists, and at least 1 of the matches is equal to the given <code><value></code></li> |
| <li><code>//@ !has <path> <value></code>: Checks <code><path></code> exists, but none of the matches equal the given <code><value></code>.</li> |
| <li><code>//@ is <path> <value></code>: Check <code><path></code> matches exactly one value, and it's equal to the given <code><value></code>.</li> |
| <li><code>//@ is <path> <value> <value>...</code>: Check that <code><path></code> matches to exactly every given <code><value></code>. |
| Ordering doesn't matter here.</li> |
| <li><code>//@ !is <path> <value></code>: Check <code><path></code> matches exactly one value, and that value is not equal to the given <code><value></code>.</li> |
| <li><code>//@ count <path> <number></code>: Check that <code><path></code> matches to <code><number></code> of values.</li> |
| <li><code>//@ set <name> = <path></code>: Check that <code><path></code> matches exactly one value, and store that value to the variable called <code><name></code>.</li> |
| </ul> |
| <p>These are defined in <a href="https://github.com/rust-lang/rust/blob/HEAD/src/tools/jsondocck/src/directive.rs"><code>directive.rs</code></a>.</p> |
| <h3 id="values"><a class="header" href="#values">Values</a></h3> |
| <p>Values can be either JSON values, or variables.</p> |
| <ul> |
| <li> |
| <p>JSON values are JSON literals, e.g. <code>true</code>, <code>"string"</code>, <code>{"key": "value"}</code>. |
| These often need to be quoted using <code>'</code>, to be processed as 1 value. See <a href="rustdoc-internals/rustdoc-json-test-suite.html#argument-spliting">§Argument spliting</a></p> |
| </li> |
| <li> |
| <p>Variables can be used to store the value in one path, and use it in later queries. |
| They are set with the <code>//@ set <name> = <path></code> directive, and accessed with <code>$<name></code></p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>//@ set foo = $some.path |
| //@ is $.some.other.path $foo |
| <span class="boring">}</span></code></pre></pre> |
| </li> |
| </ul> |
| <h3 id="argument-spliting"><a class="header" href="#argument-spliting">Argument spliting</a></h3> |
| <p>Arguments to directives are split using the <a href="https://docs.rs/shlex/1.3.0/shlex/index.html">shlex</a> crate, which implements POSIX shell escaping. |
| This is because both <code><path></code> and <code><value></code> arguments to <a href="rustdoc-internals/rustdoc-json-test-suite.html#directives">directives</a> frequently have both |
| whitespace and quote marks.</p> |
| <p>To use the <code>@ is</code> with a <code><path></code> of <code>$.index[?(@.docs == "foo")].some.field</code> and a value of <code>"bar"</code> <sup class="footnote-reference" id="fr-why_quote-1"><a href="#footnote-why_quote">1</a></sup>, you'd write:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>//@ is '$.is[?(@.docs == "foo")].some.field' '"bar"' |
| <span class="boring">}</span></code></pre></pre> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-why_quote"> |
| <p>The value needs to be <code>"bar"</code> <em>after</em> shlex splitting, because we |
| it needs to be a JSON string value. <a href="#fr-why_quote-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="stdoffload"><a class="header" href="#stdoffload">std::offload</a></h1> |
| <p>This module is under active development. Once upstream, it should allow Rust developers to run Rust code on GPUs. |
| We aim to develop a <code>rusty</code> GPU programming interface, which is safe, convenient and sufficiently fast by default. |
| This includes automatic data movement to and from the GPU, in a efficient way. We will (later) |
| also offer more advanced, possibly unsafe, interfaces which allow a higher degree of control.</p> |
| <p>The implementation is based on LLVM's "offload" project, which is already used by OpenMP to run Fortran or C++ code on GPUs. |
| While the project is under development, users will need to call other compilers like clang to finish the compilation process.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="installation"><a class="header" href="#installation">Installation</a></h1> |
| <p><code>std::offload</code> is partly available in nightly builds for users. For now, everyone however still needs to build rustc from source to use all features of it.</p> |
| <h2 id="build-instructions"><a class="header" href="#build-instructions">Build instructions</a></h2> |
| <p>First you need to clone and configure the Rust repository:</p> |
| <pre><code class="language-bash">git clone git@github.com:rust-lang/rust |
| cd rust |
| ./configure --enable-llvm-link-shared --release-channel=nightly --enable-llvm-assertions --enable-llvm-offload --enable-llvm-enzyme --enable-clang --enable-lld --enable-option-checking --enable-ninja --disable-docs |
| </code></pre> |
| <p>Afterwards you can build rustc using:</p> |
| <pre><code class="language-bash">./x build --stage 1 library |
| </code></pre> |
| <p>Afterwards rustc toolchain link will allow you to use it through cargo:</p> |
| <pre><code>rustup toolchain link offload build/host/stage1 |
| rustup toolchain install nightly # enables -Z unstable-options |
| </code></pre> |
| <h2 id="build-instruction-for-llvm-itself"><a class="header" href="#build-instruction-for-llvm-itself">Build instruction for LLVM itself</a></h2> |
| <pre><code class="language-bash">git clone git@github.com:llvm/llvm-project |
| cd llvm-project |
| mkdir build |
| cd build |
| cmake -G Ninja ../llvm -DLLVM_TARGETS_TO_BUILD="host,AMDGPU,NVPTX" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="offload,openmp" -DLLVM_ENABLE_PLUGINS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. |
| ninja |
| ninja install |
| </code></pre> |
| <p>This gives you a working LLVM build.</p> |
| <h2 id="testing"><a class="header" href="#testing">Testing</a></h2> |
| <p>run</p> |
| <pre><code>./x test --stage 1 tests/codegen-llvm/gpu_offload |
| </code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="usage"><a class="header" href="#usage">Usage</a></h1> |
| <p>This feature is work-in-progress, and not ready for usage. The instructions here are for contributors, or people interested in following the latest progress. |
| We currently work on launching the following Rust kernel on the GPU. To follow along, copy it to a <code>src/lib.rs</code> file.</p> |
| <pre><pre class="playground"><code class="language-rust">#![feature(abi_gpu_kernel)] |
| #![no_std] |
| |
| #[cfg(target_os = "linux")] |
| extern crate libc; |
| #[cfg(target_os = "linux")] |
| use libc::c_char; |
| |
| use core::mem; |
| |
| #[panic_handler] |
| fn panic(_: &core::panic::PanicInfo) -> ! { |
| loop {} |
| } |
| |
| #[cfg(target_os = "linux")] |
| #[unsafe(no_mangle)] |
| #[inline(never)] |
| fn main() { |
| let array_c: *mut [f64; 256] = |
| unsafe { libc::calloc(256, (mem::size_of::<f64>()) as libc::size_t) as *mut [f64; 256] }; |
| let output = c"The first element is zero %f\n"; |
| let output2 = c"The first element is NOT zero %f\n"; |
| let output3 = c"The second element is %f\n"; |
| unsafe { |
| let val: *const c_char = if (*array_c)[0] < 0.1 { |
| output.as_ptr() |
| } else { |
| output2.as_ptr() |
| }; |
| libc::printf(val, (*array_c)[0]); |
| } |
| |
| unsafe { |
| kernel_1(array_c); |
| } |
| core::hint::black_box(&array_c); |
| unsafe { |
| let val: *const c_char = if (*array_c)[0] < 0.1 { |
| output.as_ptr() |
| } else { |
| output2.as_ptr() |
| }; |
| libc::printf(val, (*array_c)[0]); |
| libc::printf(output3.as_ptr(), (*array_c)[1]); |
| } |
| } |
| |
| #[cfg(target_os = "linux")] |
| unsafe extern "C" { |
| pub fn kernel_1(array_b: *mut [f64; 256]); |
| } |
| |
| #[cfg(not(target_os = "linux"))] |
| #[unsafe(no_mangle)] |
| #[inline(never)] |
| pub extern "gpu-kernel" fn kernel_1(x: *mut [f64; 256]) { |
| unsafe { (*x)[0] = 21.0 }; |
| }</code></pre></pre> |
| <h2 id="compile-instructions"><a class="header" href="#compile-instructions">Compile instructions</a></h2> |
| <p>It is important to use a clang compiler build on the same llvm as rustc. Just calling clang without the full path will likely use your system clang, which probably will be incompatible. So either substitute clang/lld invocations below with absolute path, or set your <code>PATH</code> accordingly.</p> |
| <p>First we generate the host (cpu) code. The first build is just to compile libc, take note of the hashed path. Then we call rustc directly to build our host code, while providing the libc artifact to rustc.</p> |
| <pre><code>cargo +offload build -r -v |
| rustc +offload --edition 2024 src/lib.rs -g --crate-type cdylib -C opt-level=3 -C panic=abort -C lto=fat -L dependency=/absolute_path_to/target/release/deps --extern libc=/absolute_path_to/target/release/deps/liblibc-<HASH>.rlib --emit=llvm-bc,llvm-ir -Zoffload=Enable -Zunstable-options |
| </code></pre> |
| <p>Now we generate the device code. Replace the target-cpu with the right code for your gpu.</p> |
| <pre><code>RUSTFLAGS="-Ctarget-cpu=gfx90a --emit=llvm-bc,llvm-ir -Zoffload=Enable -Zunstable-options" cargo +offload build -Zunstable-options -r -v --target amdgcn-amd-amdhsa -Zbuild-std=core |
| </code></pre> |
| <pre><code>"clang-21" "-cc1" "-triple" "x86_64-unknown-linux-gnu" "-S" "-save-temps=cwd" "-disable-free" "-clear-ast-before-backend" "-main-file-name" "lib.rs" "-mrelocation-model" "pic" "-pic-level" "2" "-pic-is-pie" "-mframe-pointer=all" "-fmath-errno" "-ffp-contract=on" "-fno-rounding-math" "-mconstructor-aliases" "-funwind-tables=2" "-target-cpu" "x86-64" "-tune-cpu" "generic" "-resource-dir" "/<ABSOLUTE_PATH_TO>/rust/build/x86_64-unknown-linux-gnu/llvm/lib/clang/21" "-ferror-limit" "19" "-fopenmp" "-fopenmp-offload-mandatory" "-fgnuc-version=4.2.1" "-fskip-odr-check-in-gmf" "-fembed-offload-object=host.out" "-fopenmp-targets=amdgcn-amd-amdhsa" "-faddrsig" "-D__GCC_HAVE_DWARF2_CFI_ASM=1" "-o" "host.s" "-x" "ir" "lib.bc" |
| |
| "clang-21" "-cc1as" "-triple" "x86_64-unknown-linux-gnu" "-filetype" "obj" "-main-file-name" "lib.rs" "-target-cpu" "x86-64" "-mrelocation-model" "pic" "-o" "host.o" "host.s" |
| |
| "clang-linker-wrapper" "--should-extract=gfx90a" "--device-compiler=amdgcn-amd-amdhsa=-g" "--device-compiler=amdgcn-amd-amdhsa=-save-temps=cwd" "--device-linker=amdgcn-amd-amdhsa=-lompdevice" "--host-triple=x86_64-unknown-linux-gnu" "--save-temps" "--linker-path=/ABSOlUTE_PATH_TO/rust/build/x86_64-unknown-linux-gnu/lld/bin/ld.lld" "--hash-style=gnu" "--eh-frame-hdr" "-m" "elf_x86_64" "-pie" "-dynamic-linker" "/lib64/ld-linux-x86-64.so.2" "-o" "bare" "/lib/../lib64/Scrt1.o" "/lib/../lib64/crti.o" "/ABSOLUTE_PATH_TO/crtbeginS.o" "-L/ABSOLUTE_PATH_TO/rust/build/x86_64-unknown-linux-gnu/llvm/bin/../lib/x86_64-unknown-linux-gnu" "-L/ABSOLUTE_PATH_TO/rust/build/x86_64-unknown-linux-gnu/llvm/lib/clang/21/lib/x86_64-unknown-linux-gnu" "-L/lib/../lib64" "-L/usr/lib64" "-L/lib" "-L/usr/lib" "host.o" "-lstdc++" "-lm" "-lomp" "-lomptarget" "-L/ABSOLUTE_PATH_TO/rust/build/x86_64-unknown-linux-gnu/llvm/lib" "-lgcc_s" "-lgcc" "-lpthread" "-lc" "-lgcc_s" "-lgcc" "/ABSOLUTE_PATH_TO/crtendS.o" "/lib/../lib64/crtn.o" |
| </code></pre> |
| <p>Especially for the last three commands I recommend to not fix the paths, but rather just re-generate them by copying a bare-mode openmp example and compiling it with your clang. By adding <code>-###</code> to your clang invocation, you can see the invidual steps. |
| You can ignore other steps, e.g. the invocation of a "clang-offload-packager".</p> |
| <pre><code>myclang++ -fuse-ld=lld -O3 -fopenmp -fopenmp-offload-mandatory --offload-arch=gfx90a omp_bare.cpp -o main -### |
| </code></pre> |
| <p>In the final step, you can now run your binary</p> |
| <pre><code>./main |
| The first element is zero 0.000000 |
| The first element is NOT zero 21.000000 |
| The second element is 0.000000 |
| </code></pre> |
| <p>To receive more information about the memory transfer, you can enable info printing with</p> |
| <pre><code>LIBOMPTARGET_INFO=-1 ./main |
| </code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><p>The <code>std::autodiff</code> module in Rust allows differentiable programming:</p> |
| <pre><pre class="playground"><code class="language-rust">#![feature(autodiff)] |
| use std::autodiff::*; |
| |
| // f(x) = x * x, f'(x) = 2.0 * x |
| // bar therefore returns (x * x, 2.0 * x) |
| #[autodiff_reverse(bar, Active, Active)] |
| fn foo(x: f32) -> f32 { x * x } |
| |
| fn main() { |
| assert_eq!(bar(3.0, 1.0), (9.0, 6.0)); |
| assert_eq!(bar(4.0, 1.0), (16.0, 8.0)); |
| }</code></pre></pre> |
| <p>The detailed documentation for the <code>std::autodiff</code> module is available at <a href="https://doc.rust-lang.org/std/autodiff/index.html">std::autodiff</a>.</p> |
| <p>Differentiable programming is used in various fields like numerical computing, <a href="https://gitlab.com/micromorph/ratel">solid mechanics</a>, <a href="https://arxiv.org/abs/2411.17011">computational chemistry</a>, <a href="https://github.com/WaterLily-jl/WaterLily.jl">fluid dynamics</a> or for Neural Network training via Backpropagation, <a href="https://github.com/martinjrobins/diffsol">ODE solver</a>, <a href="https://github.com/alecjacobson/libigl-enzyme-example?tab=readme-ov-file#run">differentiable rendering</a>, <a href="https://github.com/PennyLaneAI/catalyst">quantum computing</a>, and climate simulations.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="installation-1"><a class="header" href="#installation-1">Installation</a></h1> |
| <p>In the near future, <code>std::autodiff</code> should become available in nightly builds for users. As a contributor however, you will still need to build rustc from source. Please be aware that the msvc target is not supported at the moment, all other tier 1 targets should work. Please open an issue if you encounter any problems on a supported tier 1 target, or if you successfully build this project on a tier2/tier3 target.</p> |
| <h2 id="build-instructions-1"><a class="header" href="#build-instructions-1">Build instructions</a></h2> |
| <p>First you need to clone and configure the Rust repository. Based on your preferences, you might also want to <code>--enable-clang</code> or <code>--enable-lld</code>.</p> |
| <pre><code class="language-bash">git clone git@github.com:rust-lang/rust |
| cd rust |
| ./configure --release-channel=nightly --enable-llvm-enzyme --enable-llvm-link-shared --enable-llvm-assertions --enable-ninja --enable-option-checking --disable-docs --set llvm.download-ci-llvm=false |
| </code></pre> |
| <p>Afterwards you can build rustc using:</p> |
| <pre><code class="language-bash">./x build --stage 1 library |
| </code></pre> |
| <p>Afterwards rustc toolchain link will allow you to use it through cargo:</p> |
| <pre><code>rustup toolchain link enzyme build/host/stage1 |
| rustup toolchain install nightly # enables -Z unstable-options |
| </code></pre> |
| <p>You can then run our test cases:</p> |
| <pre><code class="language-bash">./x test --stage 1 tests/codegen-llvm/autodiff |
| ./x test --stage 1 tests/pretty/autodiff |
| ./x test --stage 1 tests/ui/autodiff |
| ./x test --stage 1 tests/ui/feature-gates/feature-gate-autodiff.rs |
| </code></pre> |
| <p>Autodiff is still experimental, so if you want to use it in your own projects, you will need to add <code>lto="fat"</code> to your Cargo.toml |
| and use <code>RUSTFLAGS="-Zautodiff=Enable" cargo +enzyme</code> instead of <code>cargo</code> or <code>cargo +nightly</code>.</p> |
| <h2 id="compiler-explorer-and-dist-builds"><a class="header" href="#compiler-explorer-and-dist-builds">Compiler Explorer and dist builds</a></h2> |
| <p>Our compiler explorer instance can be updated to a newer rustc in a similar way. First, prepare a docker instance.</p> |
| <pre><code class="language-bash">docker run -it ubuntu:22.04 |
| export CC=clang CXX=clang++ |
| apt update |
| apt install wget vim python3 git curl libssl-dev pkg-config lld ninja-build cmake clang build-essential |
| </code></pre> |
| <p>Then build rustc in a slightly altered way:</p> |
| <pre><code class="language-bash">git clone https://github.com/rust-lang/rust |
| cd rust |
| ./configure --release-channel=nightly --enable-llvm-enzyme --enable-llvm-link-shared --enable-llvm-assertions --enable-ninja --enable-option-checking --disable-docs --set llvm.download-ci-llvm=false |
| ./x dist |
| </code></pre> |
| <p>We then copy the tarball to our host. The dockerid is the newest entry under <code>docker ps -a</code>.</p> |
| <pre><code class="language-bash">docker cp <dockerid>:/rust/build/dist/rust-nightly-x86_64-unknown-linux-gnu.tar.gz rust-nightly-x86_64-unknown-linux-gnu.tar.gz |
| </code></pre> |
| <p>Afterwards we can create a new (pre-release) tag on the EnzymeAD/rust repository and make a PR against the EnzymeAD/enzyme-explorer repository to update the tag. |
| Remember to ping <code>tgymnich</code> on the PR to run his update script. Note: We should archive EnzymeAD/rust and update the instructions here. The explorer should soon |
| be able to get the rustc toolchain from the official rust servers.</p> |
| <h2 id="build-instruction-for-enzyme-itself"><a class="header" href="#build-instruction-for-enzyme-itself">Build instruction for Enzyme itself</a></h2> |
| <p>Following the Rust build instruction above will build LLVMEnzyme, LLDEnzyme, and ClangEnzyme along with the Rust compiler. |
| We recommend that approach, if you just want to use any of them and have no experience with cmake. |
| However, if you prefer to just build Enzyme without Rust, then these instructions might help.</p> |
| <pre><code class="language-bash">git clone git@github.com:llvm/llvm-project |
| cd llvm-project |
| mkdir build |
| cd build |
| cmake -G Ninja ../llvm -DLLVM_TARGETS_TO_BUILD="host" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="openmp" -DLLVM_ENABLE_PLUGINS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. |
| ninja |
| ninja install |
| </code></pre> |
| <p>This gives you a working LLVM build, now we can continue with building Enzyme. |
| Leave the <code>llvm-project</code> folder, and execute the following commands:</p> |
| <pre><code class="language-bash">git clone git@github.com:EnzymeAD/Enzyme |
| cd Enzyme/enzyme |
| mkdir build |
| cd build |
| cmake .. -G Ninja -DLLVM_DIR=<YourLocalPath>/llvm-project/build/lib/cmake/llvm/ -DLLVM_EXTERNAL_LIT=<YourLocalPath>/llvm-project/llvm/utils/lit/lit.py -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=YES -DBUILD_SHARED_LIBS=ON |
| ninja |
| </code></pre> |
| <p>This will build Enzyme, and you can find it in <code>Enzyme/enzyme/build/lib/<LLD/Clang/LLVM/lib>Enzyme.so</code>. (Endings might differ based on your OS).</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="reporting-backend-crashes"><a class="header" href="#reporting-backend-crashes">Reporting backend crashes</a></h1> |
| <p>If after a compilation failure you are greeted by a large amount of llvm-ir code, then our enzyme backend likely failed to compile your code. These cases are harder to debug, so your help is highly appreciated. Please also keep in mind that release builds are usually much more likely to work at the moment.</p> |
| <p>The final goal here is to reproduce your bug in the enzyme <a href="https://enzyme.mit.edu/explorer/">compiler explorer</a>, in order to create a bug report in the <a href="https://github.com/enzymead/enzyme/issues">Enzyme</a> repository.</p> |
| <p>We have an <code>autodiff</code> flag which you can pass to <code>rustflags</code> to help with this. it will print the whole llvm-ir module, along with some <code>__enzyme_fwddiff</code> or <code>__enzyme_autodiff</code> calls. A potential workflow on linux could look like:</p> |
| <h2 id="controlling-llvm-ir-generation"><a class="header" href="#controlling-llvm-ir-generation">Controlling llvm-ir generation</a></h2> |
| <p>Before generating the llvm-ir, keep in mind two techniques that can help ensure the relevant rust code is visible for debugging:</p> |
| <ul> |
| <li><strong><code>std::hint::black_box</code></strong>: wrap rust variables or expressions in <code>std::hint::black_box()</code> to prevent rust and llvm from optimizing them away. This is useful when you need to inspect or manually manipulate specific values in the llvm-ir.</li> |
| <li><strong><code>extern "rust"</code> or <code>extern "c"</code></strong>: if you want to see how a specific function declaration is lowered to llvm-ir, you can declare it as <code>extern "rust"</code> or <code>extern "c"</code>. You can also look for existing <code>__enzyme_autodiff</code> or similar declarations within the generated module for examples.</li> |
| </ul> |
| <h2 id="1-generate-an-llvm-ir-reproducer"><a class="header" href="#1-generate-an-llvm-ir-reproducer">1) Generate an llvm-ir reproducer</a></h2> |
| <pre><code class="language-sh">RUSTFLAGS="-Z autodiff=Enable,PrintModbefore" cargo +enzyme build --release &> out.ll |
| </code></pre> |
| <p>This also captures a few warnings and info messages above and below your module. open out.ll and remove every line above <code>; moduleid = <somehash></code>. Now look at the end of the file and remove everything that's not part of llvm-ir, i.e. remove errors and warnings. The last line of your llvm-ir should now start with <code>!<somenumber> = </code>, i.e. <code>!40831 = !{i32 0, i32 1037508, i32 1037538, i32 1037559}</code> or <code>!43760 = !dilocation(line: 297, column: 5, scope: !43746)</code>.</p> |
| <p>The actual numbers will depend on your code.</p> |
| <h2 id="2-check-your-llvm-ir-reproducer"><a class="header" href="#2-check-your-llvm-ir-reproducer">2) Check your llvm-ir reproducer</a></h2> |
| <p>To confirm that your previous step worked, we will use llvm's <code>opt</code> tool. Find your path to the opt binary, with a path similar to <code><some_dir>/rust/build/<x86/arm/...-target-triple>/ci-llvm/bin/opt</code>. If you build LLVM from source, you'll likely need to replace <code>ci-llvm</code> with <code>build</code>. Also find <code>llvmenzyme-21.<so/dll/dylib></code> path, similar to <code>/rust/build/target-triple/enzyme/build/enzyme/llvmenzyme-21</code>. Please keep in mind that llvm frequently updates it's llvm backend, so the version number might be higher (20, 21, ...). Once you have both, run the following command:</p> |
| <pre><code class="language-sh"><path/to/opt> out.ll -load-pass-plugin=/path/to/build/<target-triple>/stage1/lib/libEnzyme-21.so -passes="enzyme" -enzyme-strict-aliasing=0 -s |
| </code></pre> |
| <p>This command might fail for future versions or on your system, in which case you should replace libEnzyme-21.so with LLVMEnzyme-21.so. Look at the Enzyme docs for instructions on how to build it. You might need to also adjust how to build your LLVM version.</p> |
| <p>If the previous step succeeded, you are going to see the same error that you saw when compiling your rust code with cargo.</p> |
| <p>If you fail to get the same error, please open an issue in the rust repository. If you succeed, congrats! the file is still huge, so let's automatically minimize it.</p> |
| <h2 id="3-minimize-your-llvm-ir-reproducer"><a class="header" href="#3-minimize-your-llvm-ir-reproducer">3) Minimize your llvm-ir reproducer</a></h2> |
| <p>First find your <code>llvm-extract</code> binary, it's in the same folder as your opt binary. then run:</p> |
| <pre><code class="language-sh"><path/to/llvm-extract> -s --func=<name> --recursive --rfunc="enzyme_autodiff*" --rfunc="enzyme_fwddiff*" --rfunc=<fnc_called_by_enzyme> out.ll -o mwe.ll |
| </code></pre> |
| <p>This command creates <code>mwe.ll</code>, a minimal working example.</p> |
| <p>Please adjust the name passed with the last <code>--func</code> flag. You can either apply the <code>#[no_mangle]</code> attribute to the function you differentiate, then you can replace it with the rust name. otherwise you will need to look up the mangled function name. To do that, open <code>out.ll</code> and search for <code>__enzyme_fwddiff</code> or <code>__enzyme_autodiff</code>. the first string in that function call is the name of your function. example:</p> |
| <pre><code class="language-llvm-ir">define double @enzyme_opt_helper_0(ptr %0, i64 %1, double %2) { |
| %4 = call double (...) @__enzyme_fwddiff(ptr @_zn2ad3_f217h3b3b1800bd39fde3e, metadata !"enzyme_const", ptr %0, metadata !"enzyme_const", i64 %1, metadata !"enzyme_dup", double %2, double %2) |
| ret double %4 |
| } |
| </code></pre> |
| <p>Here, <code>_zn2ad3_f217h3b3b1800bd39fde3e</code> is the correct name. make sure to not copy the leading <code>@</code>. redo step 2) by running the <code>opt</code> command again, but this time passing <code>mwe.ll</code> as the input file instead of <code>out.ll</code>. Check if this minimized example still reproduces the crash.</p> |
| <h2 id="4-optional-minimize-your-llvm-ir-reproducer-further"><a class="header" href="#4-optional-minimize-your-llvm-ir-reproducer-further">4) (Optional) Minimize your llvm-ir reproducer further.</a></h2> |
| <p>After the previous step you should have an <code>mwe.ll</code> file with ~5k loc. let's try to get it down to 50. find your <code>llvm-reduce</code> binary next to <code>opt</code> and <code>llvm-extract</code>. Copy the first line of your error message, an example could be:</p> |
| <pre><code class="language-sh">opt: /home/manuel/prog/rust/src/llvm-project/llvm/lib/ir/instructions.cpp:686: void llvm::callinst::init(llvm::functiontype*, llvm::value*, llvm::arrayref<llvm::value*>, llvm::arrayref<llvm::operandbundledeft<llvm::value*> >, const llvm::twine&): assertion `(args.size() == fty->getnumparams() || (fty->isvararg() && args.size() > fty->getnumparams())) && "calling a function with bad signature!"' failed. |
| </code></pre> |
| <p>If you just get a <code>segfault</code> there is no sensible error message and not much to do automatically, so continue to 5).<br /> |
| otherwise, create a <code>script.sh</code> file containing</p> |
| <pre><code class="language-sh">#!/bin/bash |
| <path/to/your/opt> $1 -load-pass-plugin=/path/to/llvmenzyme-19.so -passes="enzyme" \ |
| |& grep "/some/path.cpp:686: void llvm::callinst::init" |
| </code></pre> |
| <p>Experiment a bit with which error message you pass to grep. it should be long enough to make sure that the error is unique. However, for longer errors including <code>(</code> or <code>)</code> you will need to escape them correctly which can become annoying. Run</p> |
| <pre><code class="language-sh"><path/to/llvm-reduce> --test=script.sh mwe.ll |
| </code></pre> |
| <p>If you see <code>input isn't interesting! verify interesting-ness test</code>, you got the error message in script.sh wrong, you need to make sure that grep matches your actual error. If all works out, you will see a lot of iterations, ending with a new <code>reduced.ll</code> file. Verify with <code>opt</code> that you still get the same error.</p> |
| <h3 id="advanced-debugging-manual-llvm-ir-investigation"><a class="header" href="#advanced-debugging-manual-llvm-ir-investigation">Advanced debugging: manual llvm-ir investigation</a></h3> |
| <p>Once you have a minimized reproducer (<code>mwe.ll</code> or <code>reduced.ll</code>), you can delve deeper:</p> |
| <ul> |
| <li><strong>manual editing:</strong> try manually rewriting the llvm-ir. for certain issues, like those involving indirect calls, you might investigate enzyme-specific intrinsics like <code>__enzyme_virtualreverse</code>. Understanding how to use these might require consulting enzyme's documentation or source code.</li> |
| <li><strong>enzyme test cases:</strong> look for relevant test cases within the <a href="https://github.com/enzymead/enzyme/tree/main/enzyme/test">enzyme repository</a> that might demonstrate the correct usage of features or intrinsics related to your problem.</li> |
| </ul> |
| <h2 id="5-report-your-bug"><a class="header" href="#5-report-your-bug">5) Report your bug.</a></h2> |
| <p>Afterwards, you should be able to copy and paste your <code>mwe.ll</code> (or <code>reduced.ll</code>) example into our <a href="https://enzyme.mit.edu/explorer/">compiler explorer</a>.</p> |
| <ul> |
| <li>Select <code>llvm ir</code> as language and <code>opt 20</code> as compiler.</li> |
| <li>Replace the field to the right of your compiler with <code>-passes="enzyme"</code>, if it is not already set.</li> |
| <li>Hopefully, you will see once again your now familiar error.</li> |
| <li>Please use the share button to copy links to them.</li> |
| <li>Please create an issue on <a href="https://github.com/enzymead/enzyme/issues">https://github.com/enzymead/enzyme/issues</a> and share <code>mwe.ll</code> and (if you have it) <code>reduced.ll</code>, as well as links to the compiler explorer. Please feel free to also add your rust code or a link to it.</li> |
| </ul> |
| <h4 id="documenting-findings"><a class="header" href="#documenting-findings">Documenting findings</a></h4> |
| <p>some enzyme errors, like <code>"attempting to call an indirect active function whose runtime value is inactive"</code>, have historically caused confusion. If you investigate such an issue, even if you don't find a complete solution, please consider documenting your findings. If the insights are general to enzyme and not specific to its rust usage, contributing them to the main <a href="https://github.com/enzymead/www">enzyme documentation</a> is often the best first step. You can also mention your findings in the relevant enzyme github issue or propose updates to these docs if appropriate. This helps prevent others from starting from scratch.</p> |
| <p>With a clear reproducer and documentation, hopefully an enzyme developer will be able to fix your bug. Once that happens, the enzyme submodule inside the rust compiler will be updated, which should allow you to differentiate your rust code. Thanks for helping us to improve rust-ad.</p> |
| <h1 id="minimize-rust-code"><a class="header" href="#minimize-rust-code">Minimize rust code</a></h1> |
| <p>Beyond having a minimal llvm-ir reproducer, it is also helpful to have a minimal rust reproducer without dependencies. This allows us to add it as a test case to ci once we fix it, which avoids regressions for the future.</p> |
| <p>There are a few solutions to help you with minimizing the rust reproducer. This is probably the most simple automated approach: <a href="https://github.com/nilstrieb/cargo-minimize">cargo-minimize</a>.</p> |
| <p>Otherwise we have various alternatives, including <a href="https://github.com/langston-barrett/treereduce"><code>treereduce</code></a>, <a href="https://github.com/googleprojectzero/halfempty"><code>halfempty</code></a>, or <a href="https://github.com/renatahodovan/picireny"><code>picireny</code></a>, potentially also <a href="https://github.com/csmith-project/creduce"><code>creduce</code></a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="supported-rustflags"><a class="header" href="#supported-rustflags">Supported <code>RUSTFLAGS</code></a></h1> |
| <p>To support you while debugging or profiling, we have added support for an experimental <code>-Z autodiff</code> rustc flag (which can be passed to cargo via <code>RUSTFLAGS</code>), which allow changing the behaviour of Enzyme, without recompiling rustc. We currently support the following values for <code>autodiff</code>.</p> |
| <h3 id="debug-flags"><a class="header" href="#debug-flags">Debug Flags</a></h3> |
| <pre><code class="language-text">PrintTA // Print TypeAnalysis information |
| PrintTAFn // Print TypeAnalysis information for a specific function |
| PrintAA // Print ActivityAnalysis information |
| Print // Print differentiated functions while they are being generated and optimized |
| PrintPerf // Print AD related Performance warnings |
| PrintModBefore // Print the whole LLVM-IR module directly before running AD |
| PrintModAfter // Print the whole LLVM-IR module after running AD, before optimizations |
| PrintModFinal // Print the whole LLVM-IR module after running optimizations and AD |
| LooseTypes // Risk incorrect derivatives instead of aborting when missing Type Info |
| </code></pre> |
| <div class="warning"> |
| <p><code>LooseTypes</code> is often helpful to get rid of Enzyme errors stating <code>Can not deduce type of <X></code> and to be able to run some code. But please keep in mind that this flag absolutely has the chance to cause incorrect gradients. Even worse, the gradients might be correct for certain input values, but not for others. So please create issues about such bugs and only use this flag temporarily while you wait for your bug to be fixed.</p> |
| </div> |
| <h3 id="benchmark-flags"><a class="header" href="#benchmark-flags">Benchmark flags</a></h3> |
| <p>For performance experiments and benchmarking we also support</p> |
| <pre><code class="language-text">NoPostopt // We won't optimize the LLVM-IR Module after AD |
| RuntimeActivity // Enables the runtime activity feature from Enzyme |
| Inline // Instructs Enzyme to maximize inlining as far as possible, beyond LLVM's default |
| </code></pre> |
| <p>You can combine multiple <code>autodiff</code> values using a comma as separator:</p> |
| <pre><code class="language-bash">RUSTFLAGS="-Z autodiff=Enable,LooseTypes,PrintPerf" cargo +enzyme build |
| </code></pre> |
| <p>Using <code>-Zautodiff=Enable</code> will allow using autodiff and update your normal rustc compilation pipeline:</p> |
| <ol> |
| <li>Run your selected compilation pipeline. If you selected a release build, we will disable vectorization and loop unrolling.</li> |
| <li>Differentiate your functions.</li> |
| <li>Run your selected compilation pipeline again on the whole module. This time we do not disable vectorization or loop unrolling.</li> |
| </ol> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="typetrees-for-autodiff"><a class="header" href="#typetrees-for-autodiff">TypeTrees for Autodiff</a></h1> |
| <h2 id="what-are-typetrees"><a class="header" href="#what-are-typetrees">What are TypeTrees?</a></h2> |
| <p>Memory layout descriptors for Enzyme. Tell Enzyme exactly how types are structured in memory so it can compute derivatives efficiently.</p> |
| <h2 id="structure"><a class="header" href="#structure">Structure</a></h2> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(Vec<Type>) |
| |
| Type { |
| offset: isize, // byte offset (-1 = everywhere) |
| size: usize, // size in bytes |
| kind: Kind, // Float, Integer, Pointer, etc. |
| child: TypeTree // nested structure |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="example-fn-computex-f32-data-f32---f32"><a class="header" href="#example-fn-computex-f32-data-f32---f32">Example: <code>fn compute(x: &f32, data: &[f32]) -> f32</code></a></h2> |
| <p><strong>Input 0: <code>x: &f32</code></strong></p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![Type { |
| offset: -1, size: 8, kind: Pointer, |
| child: TypeTree(vec![Type { |
| offset: 0, size: 4, kind: Float, // Single value: use offset 0 |
| child: TypeTree::new() |
| }]) |
| }]) |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Input 1: <code>data: &[f32]</code></strong></p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![Type { |
| offset: -1, size: 8, kind: Pointer, |
| child: TypeTree(vec![Type { |
| offset: -1, size: 4, kind: Float, // -1 = all elements |
| child: TypeTree::new() |
| }]) |
| }]) |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Output: <code>f32</code></strong></p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![Type { |
| offset: 0, size: 4, kind: Float, // Single scalar: use offset 0 |
| child: TypeTree::new() |
| }]) |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="why-needed"><a class="header" href="#why-needed">Why Needed?</a></h2> |
| <ul> |
| <li>Enzyme can't deduce complex type layouts from LLVM IR</li> |
| <li>Prevents slow memory pattern analysis</li> |
| <li>Enables correct derivative computation for nested structures</li> |
| <li>Tells Enzyme which bytes are differentiable vs metadata</li> |
| </ul> |
| <h2 id="what-enzyme-does-with-this-information"><a class="header" href="#what-enzyme-does-with-this-information">What Enzyme Does With This Information:</a></h2> |
| <p>Without TypeTrees:</p> |
| <pre><code class="language-llvm">; Enzyme sees generic LLVM IR: |
| define float @distance(ptr %p1, ptr %p2) { |
| ; Has to guess what these pointers point to |
| ; Slow analysis of all memory operations |
| ; May miss optimization opportunities |
| } |
| </code></pre> |
| <p>With TypeTrees:</p> |
| <pre><code class="language-llvm">define "enzyme_type"="{[-1]:Float@float}" float @distance( |
| ptr "enzyme_type"="{[-1]:Pointer, [-1,0]:Float@float}" %p1, |
| ptr "enzyme_type"="{[-1]:Pointer, [-1,0]:Float@float}" %p2 |
| ) { |
| ; Enzyme knows exact type layout |
| ; Can generate efficient derivative code directly |
| } |
| </code></pre> |
| <h1 id="typetrees---offset-and--1-explained"><a class="header" href="#typetrees---offset-and--1-explained">TypeTrees - Offset and -1 Explained</a></h1> |
| <h2 id="type-structure"><a class="header" href="#type-structure">Type Structure</a></h2> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>Type { |
| offset: isize, // WHERE this type starts |
| size: usize, // HOW BIG this type is |
| kind: Kind, // WHAT KIND of data (Float, Int, Pointer) |
| child: TypeTree // WHAT'S INSIDE (for pointers/containers) |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="offset-values"><a class="header" href="#offset-values">Offset Values</a></h2> |
| <h3 id="regular-offset-0-4-8-etc"><a class="header" href="#regular-offset-0-4-8-etc">Regular Offset (0, 4, 8, etc.)</a></h3> |
| <p><strong>Specific byte position within a structure</strong></p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct Point { |
| x: f32, // offset 0, size 4 |
| y: f32, // offset 4, size 4 |
| id: i32, // offset 8, size 4 |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>TypeTree for <code>&Point</code> (internal representation):</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![ |
| Type { offset: 0, size: 4, kind: Float }, // x at byte 0 |
| Type { offset: 4, size: 4, kind: Float }, // y at byte 4 |
| Type { offset: 8, size: 4, kind: Integer } // id at byte 8 |
| ]) |
| <span class="boring">}</span></code></pre></pre> |
| <p>Generates LLVM</p> |
| <pre><code class="language-llvm">"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@float, [-1,4]:Float@float, [-1,8]:Integer, [-1,9]:Integer, [-1,10]:Integer, [-1,11]:Integer}" |
| </code></pre> |
| <h3 id="offset--1-special-everywhere"><a class="header" href="#offset--1-special-everywhere">Offset -1 (Special: "Everywhere")</a></h3> |
| <p><strong>Means "this pattern repeats for ALL elements"</strong></p> |
| <h4 id="example-1-direct-array-f32-100-no-pointer-indirection"><a class="header" href="#example-1-direct-array-f32-100-no-pointer-indirection">Example 1: Direct Array <code>[f32; 100]</code> (no pointer indirection)</a></h4> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![Type { |
| offset: -1, // ALL positions |
| size: 4, // each f32 is 4 bytes |
| kind: Float, // every element is float |
| }]) |
| <span class="boring">}</span></code></pre></pre> |
| <p>Generates LLVM: <code>"enzyme_type"="{[-1]:Float@float}"</code></p> |
| <h4 id="example-1b-array-reference-f32-100-with-pointer-indirection"><a class="header" href="#example-1b-array-reference-f32-100-with-pointer-indirection">Example 1b: Array Reference <code>&[f32; 100]</code> (with pointer indirection)</a></h4> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![Type { |
| offset: -1, size: 8, kind: Pointer, |
| child: TypeTree(vec![Type { |
| offset: -1, // ALL array elements |
| size: 4, // each f32 is 4 bytes |
| kind: Float, // every element is float |
| }]) |
| }]) |
| <span class="boring">}</span></code></pre></pre> |
| <p>Generates LLVM: <code>"enzyme_type"="{[-1]:Pointer, [-1,-1]:Float@float}"</code></p> |
| <p>Instead of listing 100 separate Types with offsets <code>0,4,8,12...396</code></p> |
| <h4 id="example-2-slice-i32"><a class="header" href="#example-2-slice-i32">Example 2: Slice <code>&[i32]</code></a></h4> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// Pointer to slice data |
| TypeTree(vec![Type { |
| offset: -1, size: 8, kind: Pointer, |
| child: TypeTree(vec![Type { |
| offset: -1, // ALL slice elements |
| size: 4, // each i32 is 4 bytes |
| kind: Integer |
| }]) |
| }]) |
| <span class="boring">}</span></code></pre></pre> |
| <p>Generates LLVM: <code>"enzyme_type"="{[-1]:Pointer, [-1,-1]:Integer}"</code></p> |
| <h4 id="example-3-mixed-structure"><a class="header" href="#example-3-mixed-structure">Example 3: Mixed Structure</a></h4> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct Container { |
| header: i64, // offset 0 |
| data: [f32; 1000], // offset 8, but elements use -1 |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>TypeTree(vec![ |
| Type { offset: 0, size: 8, kind: Integer }, // header |
| Type { offset: 8, size: 4000, kind: Pointer, |
| child: TypeTree(vec![Type { |
| offset: -1, size: 4, kind: Float // ALL array elements |
| }]) |
| } |
| ]) |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="key-distinction-single-values-vs-arrays"><a class="header" href="#key-distinction-single-values-vs-arrays">Key Distinction: Single Values vs Arrays</a></h2> |
| <p><strong>Single Values</strong> use offset <code>0</code> for precision:</p> |
| <ul> |
| <li><code>&f32</code> has exactly one f32 value at offset 0</li> |
| <li>More precise than using -1 ("everywhere")</li> |
| <li>Generates: <code>{[-1]:Pointer, [-1,0]:Float@float}</code></li> |
| </ul> |
| <p><strong>Arrays</strong> use offset <code>-1</code> for efficiency:</p> |
| <ul> |
| <li><code>&[f32; 100]</code> has the same pattern repeated 100 times</li> |
| <li>Using -1 avoids listing 100 separate offsets</li> |
| <li>Generates: <code>{[-1]:Pointer, [-1,-1]:Float@float}</code></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="source-code-representation"><a class="header" href="#source-code-representation">Source Code Representation</a></h1> |
| <p>This part describes the process of taking raw source code from the user and |
| transforming it into various forms that the compiler can work with easily. |
| These are called <em>intermediate representations (IRs)</em>.</p> |
| <p>This process starts with compiler understanding what the user has asked for: |
| parsing the command line arguments given and determining what it is to compile. |
| After that, the compiler transforms the user input into a series of IRs that |
| look progressively less like what the user wrote.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="syntax-and-the-ast"><a class="header" href="#syntax-and-the-ast">Syntax and the AST</a></h1> |
| <p>Working directly with source code is very inconvenient and error-prone. |
| Thus, before we do anything else, we convert raw source code into an |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html">Abstract Syntax Tree (AST)</a>. It turns out that doing this involves a lot of work, |
| including <a href="./the-parser.html">lexing, parsing</a>, <a href="./macro-expansion.html">macro expansion</a>, <a href="./name-resolution.html">name resolution</a>, conditional |
| compilation, <a href="./feature-gate-ck.html">feature-gate checking</a>, and <a href="./ast-validation.html">validation</a> of the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html">AST</a>. |
| In this chapter, we take a look at all of these steps.</p> |
| <p>Notably, there isn't always a clean ordering between these tasks. |
| For example, macro expansion relies on name resolution to resolve the names of macros and imports. |
| And parsing requires macro expansion, which in turn may require parsing the output of the macro.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="lexing-and-parsing-1"><a class="header" href="#lexing-and-parsing-1">Lexing and parsing</a></h1> |
| <p>The very first thing the compiler does is take the program (in UTF-8 Unicode text) |
| and turn it into a data format the compiler can work with more conveniently than strings. |
| This happens in two stages: Lexing and Parsing.</p> |
| <ol> |
| <li><em>Lexing</em> takes strings and turns them into streams of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/token/index.html">tokens</a>. For |
| example, <code>foo.bar + buz</code> would be turned into the tokens <code>foo</code>, <code>.</code>, <code>bar</code>, |
| <code>+</code>, and <code>buz</code>. This is implemented in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a>.</li> |
| </ol> |
| <ol start="2"> |
| <li><em>Parsing</em> takes streams of tokens and turns them into a structured form |
| which is easier for the compiler to work with, usually called an <a href="./ast-validation.html"><em>Abstract |
| Syntax Tree</em> (AST)</a> .</li> |
| </ol> |
| <h2 id="the-ast"><a class="header" href="#the-ast">The AST</a></h2> |
| <p>The AST mirrors the structure of a Rust program in memory, using a <code>Span</code> to |
| link a particular AST node back to its source text. The AST is defined in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/index.html"><code>rustc_ast</code></a>, along with some definitions for tokens and token |
| streams, data structures/traits for mutating ASTs, and shared definitions for |
| other AST-related parts of the compiler (like the lexer and |
| macro-expansion).</p> |
| <p>Every node in the AST has its own <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html"><code>NodeId</code></a>, including top-level items |
| such as structs, but also individual statements and expressions. A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html"><code>NodeId</code></a> |
| is an identifier number that uniquely identifies an AST node within a crate.</p> |
| <p>However, because they are absolute within a crate, adding or removing a single |
| node in the AST causes all the subsequent <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html"><code>NodeId</code></a>s to change. This renders |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html"><code>NodeId</code></a>s pretty much useless for incremental compilation, where you want as |
| few things as possible to change.</p> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html"><code>NodeId</code></a>s are used in all the <code>rustc</code> bits that operate directly on the AST, |
| like macro expansion and name resolution (more on these over the next couple chapters).</p> |
| <h2 id="parsing"><a class="header" href="#parsing">Parsing</a></h2> |
| <p>The parser is defined in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html"><code>rustc_parse</code></a>, along with a |
| high-level interface to the lexer and some validation routines that run after |
| macro expansion. In particular, the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/index.html"><code>rustc_parse::parser</code></a> contains |
| the parser implementation.</p> |
| <p>The main entrypoint to the parser is via the various <code>parse_*</code> functions and others in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html">rustc_parse</a>. They let you do things like turn a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.SourceFile.html"><code>SourceFile</code></a> |
| (e.g. the source in a single file) into a token stream, create a parser from |
| the token stream, and then execute the parser to get a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ast/struct.Crate.html"><code>Crate</code></a> (the root AST |
| node).</p> |
| <p>To minimize the amount of copying that is done, |
| both <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html"><code>Lexer</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html"><code>Parser</code></a> have lifetimes which bind them to the parent <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html"><code>ParseSess</code></a>. |
| This contains all the information needed while parsing, as well as the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html"><code>SourceMap</code></a> itself.</p> |
| <p>Note that while parsing, we may encounter macro definitions or invocations. |
| We set these aside to be expanded (see <a href="./macro-expansion.html">Macro Expansion</a>). |
| Expansion itself may require parsing the output of a macro, which may reveal more macros to be expanded, and so on.</p> |
| <h2 id="more-on-lexical-analysis"><a class="header" href="#more-on-lexical-analysis">More on lexical analysis</a></h2> |
| <p>Code for lexical analysis is split between two crates:</p> |
| <ul> |
| <li> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a> crate is responsible for breaking a <code>&str</code> into chunks |
| constituting tokens. Although it is popular to implement lexers as generated |
| finite state machines, the lexer in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a> is hand-written.</p> |
| </li> |
| <li> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html"><code>Lexer</code></a> integrates <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a> with data structures specific to |
| <code>rustc</code>. Specifically, it adds <code>Span</code> information to tokens returned by |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html"><code>rustc_lexer</code></a> and interns identifiers.</p> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="macro-expansion"><a class="header" href="#macro-expansion">Macro expansion</a></h1> |
| <p>Rust has a very powerful macro system. In the previous chapter, we saw how |
| the parser sets aside macros to be expanded (using temporary <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/placeholders/index.html">placeholders</a>). |
| This chapter is about the process of expanding those macros iteratively until |
| we have a complete <a href="./ast-validation.html"><em>Abstract Syntax Tree</em> (AST)</a> for our crate with no |
| unexpanded macros (or a compile error).</p> |
| <p>First, we discuss the algorithm that expands and integrates macro output into |
| ASTs. Next, we take a look at how hygiene data is collected. Finally, we look |
| at the specifics of expanding different types of macros.</p> |
| <p>Many of the algorithms and data structures described below are in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/index.html"><code>rustc_expand</code></a>, |
| with fundamental data structures in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/index.html"><code>rustc_expand::base</code></a>.</p> |
| <p>Also of note, <code>cfg</code> and <code>cfg_attr</code> are treated specially from other macros, and are |
| handled in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/config/index.html"><code>rustc_expand::config</code></a>.</p> |
| <h2 id="expansion-and-ast-integration"><a class="header" href="#expansion-and-ast-integration">Expansion and AST Integration</a></h2> |
| <p>Firstly, expansion happens at the crate level. Given a raw source code for |
| a crate, the compiler will produce a massive AST with all macros expanded, all |
| modules inlined, etc. The primary entry point for this process is the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.MacroExpander.html#method.fully_expand_fragment"><code>MacroExpander::fully_expand_fragment</code></a> method. With few exceptions, we |
| use this method on the whole crate (see <a href="macro-expansion.html#eager-expansion">"Eager Expansion"</a> |
| below for more detailed discussion of edge case expansion issues).</p> |
| <p>At a high level, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.MacroExpander.html#method.fully_expand_fragment"><code>fully_expand_fragment</code></a> works in iterations. We keep a |
| queue of unresolved macro invocations (i.e. macros we haven't found the |
| definition of yet). We repeatedly try to pick a macro from the queue, resolve |
| it, expand it, and integrate it back. If we can't make progress in an |
| iteration, this represents a compile error. Here is the <a href="https://github.com/rust-lang/rust/pull/53778#issuecomment-419224049">algorithm</a>:</p> |
| <ol> |
| <li>Initialize a <code>queue</code> of unresolved macros.</li> |
| <li>Repeat until <code>queue</code> is empty (or we make no progress, which is an error): |
| <ol> |
| <li><a href="./name-resolution.html">Resolve</a> imports in our partially built crate as |
| much as possible.</li> |
| <li>Collect as many macro <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.Invocation.html"><code>Invocation</code>s</a> as possible from our |
| partially built crate (<code>fn</code>-like, attributes, derives) and add them to the |
| queue.</li> |
| <li>Dequeue the first element and attempt to resolve it.</li> |
| <li>If it's resolved: |
| <ol> |
| <li>Run the macro's expander function that consumes a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html"><code>TokenStream</code></a> or |
| AST and produces a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html"><code>TokenStream</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/enum.AstFragment.html"><code>AstFragment</code></a> (depending on |
| the macro kind). (A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html"><code>TokenStream</code></a> is a collection of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/enum.TokenTree.html"><code>TokenTree</code>s</a>, |
| each of which are a token (punctuation, identifier, or literal) or a |
| delimited group (anything inside <code>()</code>/<code>[]</code>/<code>{}</code>)). |
| <ul> |
| <li>At this point, we know everything about the macro itself and can |
| call <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.LocalExpnId.html#method.set_expn_data"><code>set_expn_data</code></a> to fill in its properties in the global |
| data; that is the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/index.html">hygiene</a> data associated with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnId.html"><code>ExpnId</code></a> (see |
| <a href="macro-expansion.html#hygiene-and-hierarchies">Hygiene</a> below).</li> |
| </ul> |
| </li> |
| <li>Integrate that piece of AST into the currently-existing though |
| partially-built AST. This is essentially where the "token-like mass" |
| becomes a proper set-in-stone AST with side-tables. It happens as |
| follows: |
| <ul> |
| <li>If the macro produces tokens (e.g. a proc macro), we parse into |
| an AST, which may produce parse errors.</li> |
| <li>During expansion, we create <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html"><code>SyntaxContext</code></a>s (hierarchy 2) (see |
| <a href="macro-expansion.html#hygiene-and-hierarchies">Hygiene</a> below).</li> |
| <li>These three passes happen one after another on every AST fragment |
| freshly expanded from a macro: |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html"><code>NodeId</code></a>s are assigned by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.InvocationCollector.html"><code>InvocationCollector</code></a>. This |
| also collects new macro calls from this new AST piece and |
| adds them to the queue.</li> |
| <li><a href="hir.html#identifiers-in-the-hir">"Def paths"</a> are created and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a>s are |
| assigned to them by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/def_collector/struct.DefCollector.html"><code>DefCollector</code></a>.</li> |
| <li>Names are put into modules (from the resolver's point of |
| view) by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/build_reduced_graph/struct.BuildReducedGraphVisitor.html"><code>BuildReducedGraphVisitor</code></a>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>After expanding a single macro and integrating its output, continue |
| to the next iteration of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.MacroExpander.html#method.fully_expand_fragment"><code>fully_expand_fragment</code></a>.</li> |
| </ol> |
| </li> |
| <li>If it's not resolved: |
| <ol> |
| <li>Put the macro back in the queue.</li> |
| <li>Continue to next iteration...</li> |
| </ol> |
| </li> |
| </ol> |
| </li> |
| </ol> |
| <h3 id="error-recovery"><a class="header" href="#error-recovery">Error Recovery</a></h3> |
| <p>If we make no progress in an iteration we have reached a compilation error |
| (e.g. an undefined macro). We attempt to recover from failures (i.e. |
| unresolved macros or imports) with the intent of generating diagnostics. |
| Failure recovery happens by expanding unresolved macros into |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ast/enum.ExprKind.html#variant.Err"><code>ExprKind::Err</code></a> and allows compilation to continue past the first error |
| so that <code>rustc</code> can report more errors than just the original failure.</p> |
| <h3 id="name-resolution"><a class="header" href="#name-resolution">Name Resolution</a></h3> |
| <p>Notice that name resolution is involved here: we need to resolve imports and |
| macro names in the above algorithm. This is done in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/macros/index.html"><code>rustc_resolve::macros</code></a>, which resolves macro paths, validates |
| those resolutions, and reports various errors (e.g. "not found", "found, but |
| it's unstable", "expected x, found y"). However, we don't try to resolve |
| other names yet. This happens later, as we will see in the chapter: <a href="./name-resolution.html">Name |
| Resolution</a>.</p> |
| <h3 id="eager-expansion"><a class="header" href="#eager-expansion">Eager Expansion</a></h3> |
| <p><em>Eager expansion</em> means we expand the arguments of a macro invocation before |
| the macro invocation itself. This is implemented only for a few special |
| built-in macros that expect literals; expanding arguments first for some of |
| these macro results in a smoother user experience. As an example, consider |
| the following:</p> |
| <pre><code class="language-rust ignore">macro bar($i: ident) { $i } |
| macro foo($i: ident) { $i } |
| |
| foo!(bar!(baz));</code></pre> |
| <p>A lazy-expansion would expand <code>foo!</code> first. An eager-expansion would expand |
| <code>bar!</code> first.</p> |
| <p>Eager-expansion is not a generally available feature of Rust. Implementing |
| eager-expansion more generally would be challenging, so we implement it for a |
| few special built-in macros for the sake of user-experience. The built-in |
| macros are implemented in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_builtin_macros/index.html"><code>rustc_builtin_macros</code></a>, along with some other |
| early code generation facilities like injection of standard library imports or |
| generation of test harness. There are some additional helpers for building |
| AST fragments in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/build/index.html"><code>rustc_expand::build</code></a>. Eager-expansion generally |
| performs a subset of the things that lazy (normal) expansion does. It is done |
| by invoking <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.MacroExpander.html#method.fully_expand_fragment"><code>fully_expand_fragment</code></a> on only part of a crate (as opposed |
| to the whole crate, like we normally do).</p> |
| <h3 id="other-data-structures"><a class="header" href="#other-data-structures">Other Data Structures</a></h3> |
| <p>Here are some other notable data structures involved in expansion and |
| integration:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/trait.ResolverExpand.html"><code>ResolverExpand</code></a> - a <code>trait</code> used to break crate dependencies. This allows the |
| resolver services to be used in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/index.html"><code>rustc_ast</code></a>, despite <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/index.html"><code>rustc_resolve</code></a> and |
| pretty much everything else depending on <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/index.html"><code>rustc_ast</code></a>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/struct.ExtCtxt.html"><code>ExtCtxt</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/struct.ExpansionData.html"><code>ExpansionData</code></a> - holds various intermediate expansion |
| infrastructure data.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/enum.Annotatable.html"><code>Annotatable</code></a> - a piece of AST that can be an attribute target, almost the same |
| thing as <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/enum.AstFragment.html"><code>AstFragment</code></a> except for types and patterns that can be produced by |
| macros but cannot be annotated with attributes.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/trait.MacResult.html"><code>MacResult</code></a> - a "polymorphic" AST fragment, something that can turn into |
| a different <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/enum.AstFragment.html"><code>AstFragment</code></a> depending on its <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/enum.AstFragmentKind.html"><code>AstFragmentKind</code></a> (i.e. an item, |
| expression, pattern, etc).</li> |
| </ul> |
| <h2 id="hygiene-and-hierarchies"><a class="header" href="#hygiene-and-hierarchies">Hygiene and Hierarchies</a></h2> |
| <p>If you have ever used the C/C++ preprocessor macros, you know that there are some |
| annoying and hard-to-debug gotchas! For example, consider the following C code:</p> |
| <pre><code class="language-c">#define DEFINE_FOO struct Bar {int x;}; struct Foo {Bar bar;}; |
| |
| // Then, somewhere else |
| struct Bar { |
| ... |
| }; |
| |
| DEFINE_FOO |
| </code></pre> |
| <p>Most people avoid writing C like this – and for good reason: it doesn't |
| compile. The <code>struct Bar</code> defined by the macro clashes names with the <code>struct Bar</code> defined in the code. Consider also the following example:</p> |
| <pre><code class="language-c">#define DO_FOO(x) {\ |
| int y = 0;\ |
| foo(x, y);\ |
| } |
| |
| // Then elsewhere |
| int y = 22; |
| DO_FOO(y); |
| </code></pre> |
| <p>Do you see the problem? We wanted to generate a call <code>foo(22, 0)</code>, but instead |
| we got <code>foo(0, 0)</code> because the macro defined its own <code>y</code>!</p> |
| <p>These are both examples of <em>macro hygiene</em> issues. <em>Hygiene</em> relates to how to |
| handle names defined <em>within a macro</em>. In particular, a hygienic macro system |
| prevents errors due to names introduced within a macro. Rust macros are hygienic |
| in that they do not allow one to write the sorts of bugs above.</p> |
| <p>At a high level, hygiene within the Rust compiler is accomplished by keeping |
| track of the context where a name is introduced and used. We can then |
| disambiguate names based on that context. Future iterations of the macro system |
| will allow greater control to the macro author to use that context. For example, |
| a macro author may want to introduce a new name to the context where the macro |
| was called. Alternately, the macro author may be defining a variable for use |
| only within the macro (i.e. it should not be visible outside the macro).</p> |
| <p>The context is attached to AST nodes. All AST nodes generated by macros have |
| context attached. Additionally, there may be other nodes that have context |
| attached, such as some desugared syntax (non-macro-expanded nodes are |
| considered to just have the "root" context, as described below). |
| Throughout the compiler, we use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html"><code>rustc_span::Span</code>s</a> to refer to code locations. |
| This struct also has hygiene information attached to it, as we will see later.</p> |
| <p>Because macros invocations and definitions can be nested, the syntax context of |
| a node must be a hierarchy. For example, if we expand a macro and there is |
| another macro invocation or definition in the generated output, then the syntax |
| context should reflect the nesting.</p> |
| <p>However, it turns out that there are actually a few types of context we may |
| want to track for different purposes. Thus, there are not just one but <em>three</em> |
| expansion hierarchies that together comprise the hygiene information for a |
| crate.</p> |
| <p>All of these hierarchies need some sort of "macro ID" to identify individual |
| elements in the chain of expansions. This ID is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnId.html"><code>ExpnId</code></a>. All macros receive |
| an integer ID, assigned continuously starting from 0 as we discover new macro |
| calls. All hierarchies start at <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnId.html#method.root"><code>ExpnId::root</code></a>, which is its own |
| parent.</p> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/index.html"><code>rustc_span::hygiene</code></a> crate contains all of the hygiene-related algorithms |
| (with the exception of some hacks in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/struct.Resolver.html#method.resolve_crate_root"><code>Resolver::resolve_crate_root</code></a>) |
| and structures related to hygiene and expansion that are kept in global data.</p> |
| <p>The actual hierarchies are stored in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.HygieneData.html"><code>HygieneData</code></a>. This is a global |
| piece of data containing hygiene and expansion info that can be accessed from |
| any <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Ident.html"><code>Ident</code></a> without any context.</p> |
| <h3 id="the-expansion-order-hierarchy"><a class="header" href="#the-expansion-order-hierarchy">The Expansion Order Hierarchy</a></h3> |
| <p>The first hierarchy tracks the order of expansions, i.e., when a macro |
| invocation is in the output of another macro.</p> |
| <p>Here, the children in the hierarchy will be the "innermost" tokens. The |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnData.html"><code>ExpnData</code></a> struct itself contains a subset of properties from both macro |
| definition and macro call available through global data. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnData.html#structfield.parent"><code>ExpnData::parent</code></a> tracks the child-to-parent link in this hierarchy.</p> |
| <p>For example:</p> |
| <pre><code class="language-rust ignore">macro_rules! foo { () => { println!(); } } |
| |
| fn main() { foo!(); }</code></pre> |
| <p>In this code, the AST nodes that are finally generated would have hierarchy |
| <code>root -> id(foo) -> id(println)</code>.</p> |
| <h3 id="the-macro-definition-hierarchy"><a class="header" href="#the-macro-definition-hierarchy">The Macro Definition Hierarchy</a></h3> |
| <p>The second hierarchy tracks the order of macro definitions, i.e., when we are |
| expanding one macro another macro definition is revealed in its output. This |
| one is a bit tricky and more complex than the other two hierarchies.</p> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html"><code>SyntaxContext</code></a> represents a whole chain in this hierarchy via an ID. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContextData.html"><code>SyntaxContextData</code></a> contains data associated with the given |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html"><code>SyntaxContext</code></a>; mostly it is a cache for results of filtering that chain in |
| different ways. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContextData.html#structfield.parent"><code>SyntaxContextData::parent</code></a> is the child-to-parent |
| link here, and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContextData.html#structfield.outer_expn"><code>SyntaxContextData::outer_expns</code></a> are individual |
| elements in the chain. The "chaining-operator" is |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html#method.apply_mark"><code>SyntaxContext::apply_mark</code></a> in compiler code.</p> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html"><code>Span</code></a>, mentioned above, is actually just a compact representation of |
| a code location and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html"><code>SyntaxContext</code></a>. Likewise, an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Ident.html"><code>Ident</code></a> is just an interned |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html"><code>Symbol</code></a> + <code>Span</code> (i.e. an interned string + hygiene data).</p> |
| <p>For built-in macros, we use the context: |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html#method.apply_mark"><code>SyntaxContext::empty().apply_mark(expn_id)</code></a>, and such macros are |
| considered to be defined at the hierarchy root. We do the same for <code>proc macro</code>s because we haven't implemented cross-crate hygiene yet.</p> |
| <p>If the token had context <code>X</code> before being produced by a macro then after being |
| produced by the macro it has context <code>X -> macro_id</code>. Here are some examples:</p> |
| <p>Example 0:</p> |
| <pre><code class="language-rust ignore">macro m() { ident } |
| |
| m!();</code></pre> |
| <p>Here <code>ident</code> which initially has context <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html#method.root"><code>SyntaxContext::root</code></a> has |
| context <code>ROOT -> id(m)</code> after it's produced by <code>m</code>.</p> |
| <p>Example 1:</p> |
| <pre><code class="language-rust ignore">macro m() { macro n() { ident } } |
| |
| m!(); |
| n!();</code></pre> |
| <p>In this example the <code>ident</code> has context <code>ROOT</code> initially, then <code>ROOT -> id(m)</code> |
| after the first expansion, then <code>ROOT -> id(m) -> id(n)</code>.</p> |
| <p>Example 2:</p> |
| <p>Note that these chains are not entirely determined by their last element, in |
| other words <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnId.html"><code>ExpnId</code></a> is not isomorphic to <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.SyntaxContext.html"><code>SyntaxContext</code></a>.</p> |
| <pre><code class="language-rust ignore">macro m($i: ident) { macro n() { ($i, bar) } } |
| |
| m!(foo);</code></pre> |
| <p>After all expansions, <code>foo</code> has context <code>ROOT -> id(n)</code> and <code>bar</code> has context |
| <code>ROOT -> id(m) -> id(n)</code>.</p> |
| <p>Currently this hierarchy for tracking macro definitions is subject to the |
| so-called <a href="https://github.com/rust-lang/rust/pull/51762#issuecomment-401400732">"context transplantation hack"</a>. Modern (i.e. experimental) |
| macros have stronger hygiene than the legacy "Macros By Example" (MBE) |
| system which can result in weird interactions between the two. The hack is |
| intended to make things "just work" for now.</p> |
| <h3 id="the-call-site-hierarchy"><a class="header" href="#the-call-site-hierarchy">The Call-site Hierarchy</a></h3> |
| <p>The third and final hierarchy tracks the location of macro invocations.</p> |
| <p>In this hierarchy <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/struct.ExpnData.html#structfield.call_site"><code>ExpnData::call_site</code></a> is the <code>child -> parent</code> |
| link.</p> |
| <p>Here is an example:</p> |
| <pre><code class="language-rust ignore">macro bar($i: ident) { $i } |
| macro foo($i: ident) { $i } |
| |
| foo!(bar!(baz));</code></pre> |
| <p>For the <code>baz</code> AST node in the final output, the expansion-order hierarchy is |
| <code>ROOT -> id(foo) -> id(bar) -> baz</code>, while the call-site hierarchy is <code>ROOT -> baz</code>.</p> |
| <h3 id="macro-backtraces"><a class="header" href="#macro-backtraces">Macro Backtraces</a></h3> |
| <p>Macro backtraces are implemented in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/index.html"><code>rustc_span</code></a> using the hygiene machinery |
| in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/hygiene/index.html"><code>rustc_span::hygiene</code></a>.</p> |
| <h2 id="producing-macro-output"><a class="header" href="#producing-macro-output">Producing Macro Output</a></h2> |
| <p>Above, we saw how the output of a macro is integrated into the AST for a crate, |
| and we also saw how the hygiene data for a crate is generated. But how do we |
| actually produce the output of a macro? It depends on the type of macro.</p> |
| <p>There are two types of macros in Rust:</p> |
| <ol> |
| <li><code>macro_rules!</code> macros (a.k.a. "Macros By Example" (MBE)), and,</li> |
| <li>procedural macros (proc macros); including custom derives.</li> |
| </ol> |
| <p>During the parsing phase, the normal Rust parser will set aside the contents of |
| macros and their invocations. Later, macros are expanded using these |
| portions of the code.</p> |
| <p>Some important data structures/interfaces here:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/struct.SyntaxExtension.html"><code>SyntaxExtension</code></a> - a lowered macro representation, contains its expander |
| function, which transforms a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html"><code>TokenStream</code></a> or AST into another |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html"><code>TokenStream</code></a> or AST + some additional data like stability, or a list of |
| unstable features allowed inside the macro.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/enum.SyntaxExtensionKind.html"><code>SyntaxExtensionKind</code></a> - expander functions may have several different |
| signatures (take one token stream, or two, or a piece of AST, etc). This is |
| an <code>enum</code> that lists them.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/trait.BangProcMacro.html"><code>BangProcMacro</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/trait.TTMacroExpander.html"><code>TTMacroExpander</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/trait.AttrProcMacro.html"><code>AttrProcMacro</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/base/trait.MultiItemModifier.html"><code>MultiItemModifier</code></a> - |
| <code>trait</code>s representing the expander function signatures.</li> |
| </ul> |
| <h2 id="macros-by-example"><a class="header" href="#macros-by-example">Macros By Example</a></h2> |
| <p>MBEs have their own parser distinct from the Rust parser. When macros are |
| expanded, we may invoke the MBE parser to parse and expand a macro. The |
| MBE parser, in turn, may call the Rust parser when it needs to bind a |
| metavariable (e.g. <code>$my_expr</code>) while parsing the contents of a macro |
| invocation. The code for macro expansion is in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_expand/src/mbe"><code>compiler/rustc_expand/src/mbe/</code></a>.</p> |
| <h3 id="example"><a class="header" href="#example">Example</a></h3> |
| <pre><code class="language-rust ignore">macro_rules! printer { |
| (print $mvar:ident) => { |
| println!("{}", $mvar); |
| }; |
| (print twice $mvar:ident) => { |
| println!("{}", $mvar); |
| println!("{}", $mvar); |
| }; |
| }</code></pre> |
| <p>Here <code>$mvar</code> is called a <em>metavariable</em>. Unlike normal variables, rather than |
| binding to a value <em>at runtime</em>, a metavariable binds <em>at compile time</em> to a |
| tree of <em>tokens</em>. A <em>token</em> is a single "unit" of the grammar, such as an |
| identifier (e.g. <code>foo</code>) or punctuation (e.g. <code>=></code>). There are also other |
| special tokens, such as <code>EOF</code>, which its self indicates that there are no more |
| tokens. There are token trees resulting from the paired parentheses-like |
| characters (<code>(</code>...<code>)</code>, <code>[</code>...<code>]</code>, and <code>{</code>...<code>}</code>) – they include the open and |
| close and all the tokens in between (Rust requires that parentheses-like |
| characters be balanced). Having macro expansion operate on token streams |
| rather than the raw bytes of a source-file abstracts away a lot of complexity. |
| The macro expander (and much of the rest of the compiler) doesn't consider |
| the exact line and column of some syntactic construct in the code; it considers |
| which constructs are used in the code. Using tokens allows us to care about |
| <em>what</em> without worrying about <em>where</em>. For more information about tokens, see |
| the <a href="./the-parser.html">Parsing</a> chapter of this book.</p> |
| <pre><code class="language-rust ignore">printer!(print foo); // `foo` is a variable</code></pre> |
| <p>The process of expanding the macro invocation into the syntax tree |
| <code>println!("{}", foo)</code> and then expanding the syntax tree into a call to |
| <code>Display::fmt</code> is one common example of <em>macro expansion</em>.</p> |
| <h3 id="the-mbe-parser"><a class="header" href="#the-mbe-parser">The MBE parser</a></h3> |
| <p>There are two parts to MBE expansion done by the macro parser:</p> |
| <ol> |
| <li>parsing the definition, and,</li> |
| <li>parsing the invocations.</li> |
| </ol> |
| <p>We think of the MBE parser as a nondeterministic finite automaton (NFA) based |
| regex parser since it uses an algorithm similar in spirit to the <a href="https://en.wikipedia.org/wiki/Earley_parser">Earley |
| parsing algorithm</a>. The macro |
| parser is defined in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/mbe/macro_parser"><code>compiler/rustc_expand/src/mbe/macro_parser.rs</code></a>.</p> |
| <p>The interface of the macro parser is as follows (this is slightly simplified):</p> |
| <pre><code class="language-rust ignore">fn parse_tt( |
| &mut self, |
| parser: &mut Cow<'_, Parser<'_>>, |
| matcher: &[MatcherLoc] |
| ) -> ParseResult</code></pre> |
| <p>We use these items in macro parser:</p> |
| <ul> |
| <li>a <code>parser</code> variable is a reference to the state of a normal Rust parser, |
| including the token stream and parsing session. The token stream is what we |
| are about to ask the MBE parser to parse. We will consume the raw stream of |
| tokens and output a binding of metavariables to corresponding token trees. |
| The parsing session can be used to report parser errors.</li> |
| <li>a <code>matcher</code> variable is a sequence of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/mbe/macro_parser/enum.MatcherLoc.html"><code>MatcherLoc</code></a>s that we want to match the token stream |
| against. They're converted from the original token trees in the macro's definition before |
| matching.</li> |
| </ul> |
| <p>In the analogy of a regex parser, the token stream is the input and we are |
| matching it against the pattern defined by matcher. Using our examples, the |
| token stream could be the stream of tokens containing the inside of the example |
| invocation <code>print foo</code>, while matcher might be the sequence of token (trees) |
| <code>print $mvar:ident</code>.</p> |
| <p>The output of the parser is a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/mbe/macro_parser/enum.ParseResult.html"><code>ParseResult</code></a>, which indicates which of |
| three cases has occurred:</p> |
| <ul> |
| <li><strong>Success</strong>: the token stream matches the given matcher and we have produced a |
| binding from metavariables to the corresponding token trees.</li> |
| <li><strong>Failure</strong>: the token stream does not match matcher and results in an error |
| message such as "No rule expected token ...".</li> |
| <li><strong>Error</strong>: some fatal error has occurred <em>in the parser</em>. For example, this |
| happens if there is more than one pattern match, since that indicates the |
| macro is ambiguous.</li> |
| </ul> |
| <p>The full interface is defined <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/mbe/macro_parser/struct.TtParser.html#method.parse_tt">here</a>.</p> |
| <p>The macro parser does pretty much exactly the same as a normal regex parser |
| with one exception: in order to parse different types of metavariables, such as |
| <code>ident</code>, <code>block</code>, <code>expr</code>, etc., the macro parser must call back to the normal |
| Rust parser.</p> |
| <p>The code to parse macro definitions is in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/mbe/macro_rules"><code>compiler/rustc_expand/src/mbe/macro_rules.rs</code></a>. |
| For more information about the macro parser's implementation, see the comments in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/mbe/macro_parser"><code>compiler/rustc_expand/src/mbe/macro_parser.rs</code></a>.</p> |
| <p>Using our example, we would try to match the token stream <code>print foo</code> from the invocation against |
| the matchers <code>print $mvar:ident</code> and <code>print twice $mvar:ident</code> that we previously extracted from the |
| rules in the macro definition. When the macro parser comes to a place in the current matcher where |
| it needs to match a <em>non-terminal</em> (e.g. <code>$mvar:ident</code>), it calls back to the normal Rust parser to |
| get the contents of that non-terminal. In this case, the Rust parser would look for an <code>ident</code> |
| token, which it finds (<code>foo</code>) and returns to the macro parser. Then, the macro parser continues |
| parsing.</p> |
| <p>Note that exactly one of the matchers from the various rules should match the invocation; if there is |
| more than one match, the parse is ambiguous, while if there are no matches at all, there is a syntax |
| error.</p> |
| <p>Assuming exactly one rule matches, macro expansion will then <em>transcribe</em> the right-hand side of the |
| rule, substituting the values of any matches it captured when matching against the left-hand side.</p> |
| <h2 id="procedural-macros"><a class="header" href="#procedural-macros">Procedural Macros</a></h2> |
| <p>Procedural macros are also expanded during parsing. However, rather than |
| having a parser in the compiler, proc macros are implemented as custom, |
| third-party crates. The compiler will compile the proc macro crate and |
| specially annotated functions in them (i.e. the proc macro itself), passing |
| them a stream of tokens. A proc macro can then transform the token stream and |
| output a new token stream, which is synthesized into the AST.</p> |
| <p>The token stream type used by proc macros is <em>stable</em>, so <code>rustc</code> does not |
| use it internally. The compiler's (unstable) token stream is defined in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html"><code>rustc_ast::tokenstream::TokenStream</code></a>. This is converted into the |
| stable <a href="https://doc.rust-lang.org/proc_macro/struct.TokenStream.html"><code>proc_macro::TokenStream</code></a> and back in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/proc_macro/index.html"><code>rustc_expand::proc_macro</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/proc_macro_server/index.html"><code>rustc_expand::proc_macro_server</code></a>. |
| Since the Rust ABI is currently unstable, we use the C ABI for this conversion.</p> |
| <!-- TODO(rylev): more here. [#1160](https://github.com/rust-lang/rustc-dev-guide/issues/1160) --> |
| <h3 id="custom-derive"><a class="header" href="#custom-derive">Custom Derive</a></h3> |
| <p>Custom derives are a special type of proc macro.</p> |
| <h3 id="macros-by-example-and-macros-20"><a class="header" href="#macros-by-example-and-macros-20">Macros By Example and Macros 2.0</a></h3> |
| <p>There is an legacy and mostly undocumented effort to improve the MBE system |
| by giving it more hygiene-related features, better scoping and visibility |
| rules, etc. Internally this uses the same machinery as today's MBEs with some |
| additional syntactic sugar and are allowed to be in namespaces.</p> |
| <!-- TODO(rylev): more? [#1160](https://github.com/rust-lang/rustc-dev-guide/issues/1160) --> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="name-resolution-1"><a class="header" href="#name-resolution-1">Name resolution</a></h1> |
| <p>In the previous chapters, we saw how the <a href="./ast-validation.html"><em>Abstract Syntax Tree</em> (<code>AST</code>)</a> |
| is built with all macros expanded. We saw how doing that requires doing some |
| name resolution to resolve imports and macro names. In this chapter, we show |
| how this is actually done and more.</p> |
| <p>In fact, we don't do full name resolution during macro expansion -- we only |
| resolve imports and macros at that time. This is required to know what to even |
| expand. Later, after we have the whole AST, we do full name resolution to |
| resolve all names in the crate. This happens in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/index.html"><code>rustc_resolve::late</code></a>. |
| Unlike during macro expansion, in this late expansion, we only need to try to |
| resolve a name once, since no new names can be added. If we fail to resolve a |
| name, then it is a compiler error.</p> |
| <p>Name resolution is complex. There are different namespaces (e.g. |
| macros, values, types, lifetimes), and names may be valid at different (nested) |
| scopes. Also, different types of names can fail resolution differently, and |
| failures can happen differently at different scopes. For example, in a module |
| scope, failure means no unexpanded macros and no unresolved glob imports in |
| that module. On the other hand, in a function body scope, failure requires that a |
| name be absent from the block we are in, all outer scopes, and the global |
| scope.</p> |
| <h2 id="basics"><a class="header" href="#basics">Basics</a></h2> |
| <p>In our programs we refer to variables, types, functions, etc, by giving them |
| a name. These names are not always unique. For example, take this valid Rust |
| program:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type x = u32; |
| let x: x = 1; |
| let y: x = 2; |
| <span class="boring">}</span></code></pre></pre> |
| <p>How do we know on line 3 whether <code>x</code> is a type (<code>u32</code>) or a value (1)? These |
| conflicts are resolved during name resolution. In this specific case, name |
| resolution defines that type names and variable names live in separate |
| namespaces and therefore can co-exist.</p> |
| <p>The name resolution in Rust is a two-phase process. In the first phase, which runs |
| during <code>macro</code> expansion, we build a tree of modules and resolve imports. Macro |
| expansion and name resolution communicate with each other via the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast_lowering/trait.ResolverAstLoweringExt.html"><code>ResolverAstLoweringExt</code></a> trait.</p> |
| <p>The input to the second phase is the syntax tree, produced by parsing input |
| files and expanding <code>macros</code>. This phase produces links from all the names in the |
| source to relevant places where the name was introduced. It also generates |
| helpful error messages, like typo suggestions, traits to import or lints about |
| unused items.</p> |
| <p>A successful run of the second phase (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/struct.Resolver.html#method.resolve_crate"><code>Resolver::resolve_crate</code></a>) creates kind |
| of an index the rest of the compilation may use to ask about the present names |
| (through the <code>hir::lowering::Resolver</code> interface).</p> |
| <p>The name resolution lives in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/index.html"><code>rustc_resolve</code></a> crate, with the bulk in |
| <code>lib.rs</code> and some helpers or symbol-type specific logic in the other modules.</p> |
| <h2 id="namespaces"><a class="header" href="#namespaces">Namespaces</a></h2> |
| <p>Different kind of symbols live in different namespaces ‒ e.g. types don't |
| clash with variables. This usually doesn't happen, because variables start with |
| lower-case letter while types with upper-case one, but this is only a |
| convention. This is legal Rust code that will compile (with warnings):</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type x = u32; |
| let x: x = 1; |
| let y: x = 2; // See? x is still a type here. |
| <span class="boring">}</span></code></pre></pre> |
| <p>To cope with this, and with slightly different scoping rules for these |
| namespaces, the resolver keeps them separated and builds separate structures for |
| them.</p> |
| <p>In other words, when the code talks about namespaces, it doesn't mean the module |
| hierarchy, it's types vs. values vs. macros.</p> |
| <h2 id="scopes-and-ribs"><a class="header" href="#scopes-and-ribs">Scopes and ribs</a></h2> |
| <p>A name is visible only in certain area in the source code. This forms a |
| hierarchical structure, but not necessarily a simple one ‒ if one scope is |
| part of another, it doesn't mean a name visible in the outer scope is also |
| visible in the inner scope, or that it refers to the same thing.</p> |
| <p>To cope with that, the compiler introduces the concept of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a>s. This is |
| an abstraction of a scope. Every time the set of visible names potentially changes, |
| a new <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a> is pushed onto a stack. The places where this can happen include for |
| example:</p> |
| <ul> |
| <li>The obvious places ‒ curly braces enclosing a block, function boundaries, |
| modules.</li> |
| <li>Introducing a <code>let</code> binding ‒ this can shadow another binding with the same |
| name.</li> |
| <li>Macro expansion border ‒ to cope with macro hygiene.</li> |
| </ul> |
| <p>When searching for a name, the stack of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.LateResolutionVisitor.html#structfield.ribs"><code>ribs</code></a> is traversed from the innermost |
| outwards. This helps to find the closest meaning of the name (the one not |
| shadowed by anything else). The transition to outer <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a> may also affect |
| what names are usable ‒ if there are nested functions (not closures), |
| the inner one can't access parameters and local bindings of the outer one, |
| even though they should be visible by ordinary scoping rules. An example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn do_something<T: Default>(val: T) { // <- New rib in both types and values (1) |
| // `val` is accessible, as is the helper function |
| // `T` is accessible |
| let helper = || { // New rib on the block (2) |
| // `val` is accessible here |
| }; // End of (2), new rib on `helper` (3) |
| // `val` is accessible, `helper` variable shadows `helper` function |
| fn helper() { // <- New rib in both types and values (4) |
| // `val` is not accessible here, (4) is not transparent for locals |
| // `T` is not accessible here |
| } // End of (4) |
| let val = T::default(); // New rib (5) |
| // `val` is the variable, not the parameter here |
| } // End of (5), (3) and (1) |
| <span class="boring">}</span></code></pre></pre> |
| <p>Because the rules for different namespaces are a bit different, each namespace |
| has its own independent <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a> stack that is constructed in parallel to the others. |
| In addition, there's also a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a> stack for local labels (e.g. names of loops or |
| blocks), which isn't a full namespace in its own right.</p> |
| <h2 id="overall-strategy"><a class="header" href="#overall-strategy">Overall strategy</a></h2> |
| <p>To perform the name resolution of the whole crate, the syntax tree is traversed |
| top-down and every encountered name is resolved. This works for most kinds of |
| names, because at the point of use of a name it is already introduced in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a> |
| hierarchy.</p> |
| <p>There are some exceptions to this. Items are bit tricky, because they can be |
| used even before encountered ‒ therefore every block needs to be first scanned |
| for items to fill in its <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html"><code>Rib</code></a>.</p> |
| <p>Other, even more problematic ones, are imports which need recursive fixed-point |
| resolution and macros, that need to be resolved and expanded before the rest of |
| the code can be processed.</p> |
| <p>Therefore, the resolution is performed in multiple stages.</p> |
| <h2 id="speculative-crate-loading"><a class="header" href="#speculative-crate-loading">Speculative crate loading</a></h2> |
| <p>To give useful errors, rustc suggests importing paths into scope if they're |
| not found. How does it do this? It looks through every module of every crate |
| and looks for possible matches. This even includes crates that haven't yet |
| been loaded!</p> |
| <p>Eagerly loading crates to include import suggestions that haven't yet been |
| loaded is called <em>speculative crate loading</em>, because any errors it encounters |
| shouldn't be reported: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/index.html"><code>rustc_resolve</code></a> decided to load them, not the user. The function |
| that does this is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/struct.Resolver.html#method.lookup_import_candidates"><code>lookup_import_candidates</code></a> and lives in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/diagnostics/index.html"><code>rustc_resolve::diagnostics</code></a>.</p> |
| <p>To tell the difference between speculative loads and loads initiated by the |
| user, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/index.html"><code>rustc_resolve</code></a> passes around a <code>record_used</code> parameter, which is <code>false</code> when |
| the load is speculative.</p> |
| <h2 id="todo-16"><a class="header" href="#todo-16">TODO: <a href="https://github.com/rust-lang/rustc-dev-guide/issues/16">#16</a></a></h2> |
| <p>This is a result of the first pass of learning the code. It is definitely |
| incomplete and not detailed enough. It also might be inaccurate in places. |
| Still, it probably provides useful first guidepost to what happens in there.</p> |
| <ul> |
| <li>What exactly does it link to and how is that published and consumed by |
| following stages of compilation?</li> |
| <li>Who calls it and how it is actually used.</li> |
| <li>Is it a pass and then the result is only used, or can it be computed |
| incrementally?</li> |
| <li>The overall strategy description is a bit vague.</li> |
| <li>Where does the name <code>Rib</code> come from?</li> |
| <li>Does this thing have its own tests, or is it tested only as part of some e2e |
| testing?</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="attributes"><a class="header" href="#attributes">Attributes</a></h1> |
| <p>Attributes come in two types: <em>inert</em> (or <em>built-in</em>) and <em>active</em> (<em>non-builtin</em>).</p> |
| <h2 id="builtininert-attributes"><a class="header" href="#builtininert-attributes">Builtin/inert attributes</a></h2> |
| <p>These attributes are defined in the compiler itself, in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_feature/builtin_attrs/index.html"><code>compiler/rustc_feature/src/builtin_attrs.rs</code></a>.</p> |
| <p>Examples include <code>#[allow]</code> and <code>#[macro_use]</code>.</p> |
| <p>These attributes have several important characteristics:</p> |
| <ul> |
| <li>They are always in scope, and do not participate in typical path-based resolution.</li> |
| <li>They cannot be renamed. For example, <code>use allow as foo</code> will compile, but writing <code>#[foo]</code> will |
| produce an error.</li> |
| <li>They are 'inert', meaning they are left as-is by the macro expansion code. |
| As a result, any behavior comes as a result of the compiler explicitly checking for their presence. |
| For example, lint-related code explicitly checks for <code>#[allow]</code>, <code>#[warn]</code>, <code>#[deny]</code>, and |
| <code>#[forbid]</code>, rather than the behavior coming from the expansion of the attributes themselves.</li> |
| </ul> |
| <h2 id="non-builtinactive-attributes"><a class="header" href="#non-builtinactive-attributes">'Non-builtin'/'active' attributes</a></h2> |
| <p>These attributes are defined by a crate - either the standard library, or a proc-macro crate.</p> |
| <p><strong>Important</strong>: Many non-builtin attributes, such as <code>#[derive]</code>, are still considered part of the |
| core Rust language. However, they are <strong>not</strong> called 'builtin attributes', since they have a |
| corresponding definition in the standard library.</p> |
| <p>Definitions of non-builtin attributes take two forms:</p> |
| <ol> |
| <li>Proc-macro attributes, defined via a function annotated with <code>#[proc_macro_attribute]</code> in a |
| proc-macro crate.</li> |
| <li>AST-based attributes, defined in the standard library. These attributes have special 'stub' |
| macros defined in places like <a href="https://github.com/rust-lang/rust/blob/HEAD/library/core/src/macros/mod.rs"><code>library/core/src/macros/mod.rs</code></a>.</li> |
| </ol> |
| <p>These definitions exist to allow the macros to participate in typical path-based resolution - they |
| can be imported, re-exported, and renamed just like any other item definition. However, the body of |
| the definition is empty. Instead, the macro is annotated with the <code>#[rustc_builtin_macro]</code> |
| attribute, which tells the compiler to run a corresponding function in <code>rustc_builtin_macros</code>.</p> |
| <p>All non-builtin attributes have the following characteristics:</p> |
| <ul> |
| <li>Like all other definitions (e.g. structs), they must be brought into scope via an import. |
| Many standard library attributes are included in the prelude - this is why writing <code>#[derive]</code> |
| works without an import.</li> |
| <li>They participate in macro expansion. The implementation of the macro may leave the attribute |
| target unchanged, modify the target, produce new AST nodes, or remove the target entirely.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-test-attribute"><a class="header" href="#the-test-attribute">The <code>#[test]</code> attribute</a></h1> |
| <p>Many Rust programmers rely on a built-in attribute called <code>#[test]</code>. All |
| you have to do is mark a function and include some asserts like so:</p> |
| <pre><code class="language-rust ignore">#[test] |
| fn my_test() { |
| assert!(2+2 == 4); |
| }</code></pre> |
| <p>When this program is compiled using <code>rustc --test</code> or <code>cargo test</code>, it will |
| produce an executable that can run this, and any other test function. This |
| method of testing allows tests to live alongside code in an organic way. You |
| can even put tests inside private modules:</p> |
| <pre><code class="language-rust ignore">mod my_priv_mod { |
| fn my_priv_func() -> bool {} |
| |
| #[test] |
| fn test_priv_func() { |
| assert!(my_priv_func()); |
| } |
| }</code></pre> |
| <p>Private items can thus be easily tested without worrying about how to expose |
| them to any sort of external testing apparatus. This is key to the |
| ergonomics of testing in Rust. Semantically, however, it's rather odd. |
| How does any sort of <code>main</code> function invoke these tests if they're not visible? |
| What exactly is <code>rustc --test</code> doing?</p> |
| <p><code>#[test]</code> is implemented as a syntactic transformation inside the compiler's |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_ast"><code>rustc_ast</code></a>. Essentially, it's a fancy <a href="./macro-expansion.html"><code>macro</code></a> that |
| rewrites the crate in 3 steps:</p> |
| <h2 id="step-1-re-exporting"><a class="header" href="#step-1-re-exporting">Step 1: Re-Exporting</a></h2> |
| <p>As mentioned earlier, tests can exist inside private modules, so we need a |
| way of exposing them to the main function, without breaking any existing |
| code. To that end, <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_ast"><code>rustc_ast</code></a> will create local modules called |
| <code>__test_reexports</code> that recursively reexport tests. This expansion translates |
| the above example into:</p> |
| <pre><code class="language-rust ignore">mod my_priv_mod { |
| fn my_priv_func() -> bool {} |
| |
| pub fn test_priv_func() { |
| assert!(my_priv_func()); |
| } |
| |
| pub mod __test_reexports { |
| pub use super::test_priv_func; |
| } |
| }</code></pre> |
| <p>Now, our test can be accessed as |
| <code>my_priv_mod::__test_reexports::test_priv_func</code>. For deeper module |
| structures, <code>__test_reexports</code> will reexport modules that contain tests, so a |
| test at <code>a::b::my_test</code> becomes |
| <code>a::__test_reexports::b::__test_reexports::my_test</code>. While this process seems |
| pretty safe, what happens if there is an existing <code>__test_reexports</code> module? |
| The answer: nothing.</p> |
| <p>To explain, we need to understand how Rust's <a href="./ast-validation.html">Abstract Syntax Tree</a> |
| represents <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Ident.html">identifiers</a>. The name of every function, variable, module, |
| etc. is not stored as a string, but rather as an opaque <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html">Symbol</a> which |
| is essentially an ID number for each identifier. The compiler keeps a separate |
| hashtable that allows us to recover the human-readable name of a Symbol when |
| necessary (such as when printing a syntax error). When the compiler generates |
| the <code>__test_reexports</code> module, it generates a new <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html">Symbol</a> for the |
| identifier, so while the compiler-generated <code>__test_reexports</code> may share a name |
| with your hand-written one, it will not share a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html">Symbol</a>. This |
| technique prevents name collision during code generation and is the foundation |
| of Rust's <a href="./macro-expansion.html"><code>macro</code></a> hygiene.</p> |
| <h2 id="step-2-harness-generation"><a class="header" href="#step-2-harness-generation">Step 2: Harness generation</a></h2> |
| <p>Now that our tests are accessible from the root of our crate, we need to do |
| something with them using <a href="./ast-validation.html"><code>rustc_ast</code></a> generates a module like so:</p> |
| <pre><code class="language-rust ignore">#[main] |
| pub fn main() { |
| extern crate test; |
| test::test_main_static(&[&path::to::test1, /*...*/]); |
| }</code></pre> |
| <p>Here <code>path::to::test1</code> is a constant of type <a href="https://doc.rust-lang.org/test/struct.TestDescAndFn.html"><code>test::TestDescAndFn</code></a>.</p> |
| <p>While this transformation is simple, it gives us a lot of insight into how |
| tests are actually run. The tests are aggregated into an array and passed to |
| a test runner called <code>test_main_static</code>. We'll come back to exactly what |
| <a href="https://doc.rust-lang.org/test/struct.TestDescAndFn.html"><code>TestDescAndFn</code></a> is, but for now, the key takeaway is that there is a crate |
| called <a href="https://doc.rust-lang.org/test/index.html"><code>test</code></a> that is part of Rust core, that implements all of the |
| runtime for testing. <a href="https://doc.rust-lang.org/test/index.html"><code>test</code></a>'s interface is unstable, so the only stable way |
| to interact with it is through the <code>#[test]</code> macro.</p> |
| <h2 id="step-3-test-object-generation"><a class="header" href="#step-3-test-object-generation">Step 3: Test object generation</a></h2> |
| <p>If you've written tests in Rust before, you may be familiar with some of the |
| optional attributes available on test functions. For example, a test can be |
| annotated with <code>#[should_panic]</code> if we expect the test to cause a panic. It |
| looks something like this:</p> |
| <pre><code class="language-rust ignore">#[test] |
| #[should_panic] |
| fn foo() { |
| panic!("intentional"); |
| }</code></pre> |
| <p>This means our tests are more than just simple functions, they have |
| configuration information as well. <code>test</code> encodes this configuration data into |
| a <code>struct</code> called <a href="https://doc.rust-lang.org/test/struct.TestDesc.html"><code>TestDesc</code></a>. For each test function in a crate, |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_ast"><code>rustc_ast</code></a> will parse its attributes and generate a <a href="https://doc.rust-lang.org/test/struct.TestDesc.html"><code>TestDesc</code></a> |
| instance. It then combines the <a href="https://doc.rust-lang.org/test/struct.TestDesc.html"><code>TestDesc</code></a> and test function into the |
| predictably named <a href="https://doc.rust-lang.org/test/struct.TestDescAndFn.html"><code>TestDescAndFn</code></a> <code>struct</code>, that <a href="https://doc.rust-lang.org/test/fn.test_main_static.html"><code>test_main_static</code></a> |
| operates on. |
| For a given test, the generated <a href="https://doc.rust-lang.org/test/struct.TestDescAndFn.html"><code>TestDescAndFn</code></a> instance looks like so:</p> |
| <pre><code class="language-rust ignore">self::test::TestDescAndFn{ |
| desc: self::test::TestDesc{ |
| name: self::test::StaticTestName("foo"), |
| ignore: false, |
| should_panic: self::test::ShouldPanic::Yes, |
| allow_fail: false, |
| }, |
| testfn: self::test::StaticTestFn(|| |
| self::test::assert_test_result(::crate::__test_reexports::foo())), |
| }</code></pre> |
| <p>Once we've constructed an array of these test objects, they're passed to the |
| test runner via the harness generated in Step 2.</p> |
| <h2 id="inspecting-the-generated-code"><a class="header" href="#inspecting-the-generated-code">Inspecting the generated code</a></h2> |
| <p>On <code>nightly</code> <code>rustc</code>, there's an unstable flag called <code>unpretty</code> that you can use |
| to print out the module source after <a href="./macro-expansion.html"><code>macro</code></a> expansion:</p> |
| <pre><code class="language-bash">$ rustc my_mod.rs -Z unpretty=hir |
| </code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="panicking-in-rust"><a class="header" href="#panicking-in-rust">Panicking in Rust</a></h1> |
| <h2 id="step-1-invocation-of-the-panic-macro"><a class="header" href="#step-1-invocation-of-the-panic-macro">Step 1: Invocation of the <code>panic!</code> macro.</a></h2> |
| <p>There are actually two panic macros - one defined in <code>core</code>, and one defined in <code>std</code>. |
| This is due to the fact that code in <code>core</code> can panic. <code>core</code> is built before <code>std</code>, |
| but we want panics to use the same machinery at runtime, whether they originate in <code>core</code> |
| or <code>std</code>.</p> |
| <h3 id="core-definition-of-panic"><a class="header" href="#core-definition-of-panic">core definition of panic!</a></h3> |
| <p>The <code>core</code> <code>panic!</code> macro eventually makes the following call (in <code>library/core/src/panicking.rs</code>):</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// NOTE This function never crosses the FFI boundary; it's a Rust-to-Rust call |
| extern "Rust" { |
| #[lang = "panic_impl"] |
| fn panic_impl(pi: &PanicInfo<'_>) -> !; |
| } |
| |
| let pi = PanicInfo::internal_constructor(Some(&fmt), location); |
| unsafe { panic_impl(&pi) } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Actually resolving this goes through several layers of indirection:</p> |
| <ol> |
| <li> |
| <p>In <code>compiler/rustc_middle/src/middle/weak_lang_items.rs</code>, <code>panic_impl</code> is |
| declared as 'weak lang item', with the symbol <code>rust_begin_unwind</code>. This is |
| used in <code>rustc_hir_analysis/src/collect.rs</code> to set the actual symbol name to |
| <code>rust_begin_unwind</code>.</p> |
| <p>Note that <code>panic_impl</code> is declared in an <code>extern "Rust"</code> block, |
| which means that core will attempt to call a foreign symbol called <code>rust_begin_unwind</code> |
| (to be resolved at link time)</p> |
| </li> |
| <li> |
| <p>In <code>library/std/src/panicking.rs</code>, we have this definition:</p> |
| </li> |
| </ol> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>/// Entry point of panic from the core crate. |
| #[cfg(not(test))] |
| #[panic_handler] |
| #[unwind(allowed)] |
| pub fn begin_panic_handler(info: &PanicInfo<'_>) -> ! { |
| ... |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>The special <code>panic_handler</code> attribute is resolved via <code>compiler/rustc_middle/src/middle/lang_items</code>. |
| The <code>extract</code> function converts the <code>panic_handler</code> attribute to a <code>panic_impl</code> lang item.</p> |
| <p>Now, we have a matching <code>panic_handler</code> lang item in the <code>std</code>. This function goes |
| through the same process as the <code>extern { fn panic_impl }</code> definition in <code>core</code>, ending |
| up with a symbol name of <code>rust_begin_unwind</code>. At link time, the symbol reference in <code>core</code> |
| will be resolved to the definition of <code>std</code> (the function called <code>begin_panic_handler</code> in the |
| Rust source).</p> |
| <p>Thus, control flow will pass from core to std at runtime. This allows panics from <code>core</code> |
| to go through the same infrastructure that other panics use (panic hooks, unwinding, etc)</p> |
| <h3 id="std-implementation-of-panic"><a class="header" href="#std-implementation-of-panic">std implementation of panic!</a></h3> |
| <p>This is where the actual panic-related logic begins. In <code>library/std/src/panicking.rs</code>, |
| control passes to <code>rust_panic_with_hook</code>. This method is responsible |
| for invoking the global panic hook, and checking for double panics. Finally, |
| we call <code>__rust_start_panic</code>, which is provided by the panic runtime.</p> |
| <p>The call to <code>__rust_start_panic</code> is very weird - it is passed a <code>*mut &mut dyn PanicPayload</code>, |
| converted to an <code>usize</code>. Let's break this type down:</p> |
| <ol> |
| <li> |
| <p><code>PanicPayload</code> is an internal trait. It is implemented for <code>PanicPayload</code> |
| (a wrapper around the user-supplied payload type), and has a method |
| <code>fn take_box(&mut self) -> *mut (dyn Any + Send)</code>. |
| This method takes the user-provided payload (<code>T: Any + Send</code>), |
| boxes it, and converts the box to a raw pointer.</p> |
| </li> |
| <li> |
| <p>When we call <code>__rust_start_panic</code>, we have an <code>&mut dyn PanicPayload</code>. |
| However, this is a fat pointer (twice the size of a <code>usize</code>). |
| To pass this to the panic runtime across an FFI boundary, we take a mutable |
| reference <em>to this mutable reference</em> (<code>&mut &mut dyn PanicPayload</code>), and convert it to a raw |
| pointer (<code>*mut &mut dyn PanicPayload</code>). The outer raw pointer is a thin pointer, since it points to |
| a <code>Sized</code> type (a mutable reference). Therefore, we can convert this thin pointer into a <code>usize</code>, |
| which is suitable for passing across an FFI boundary.</p> |
| </li> |
| </ol> |
| <p>Finally, we call <code>__rust_start_panic</code> with this <code>usize</code>. We have now entered the panic runtime.</p> |
| <h2 id="step-2-the-panic-runtime"><a class="header" href="#step-2-the-panic-runtime">Step 2: The panic runtime</a></h2> |
| <p>Rust provides two panic runtimes: <code>panic_abort</code> and <code>panic_unwind</code>. The user chooses |
| between them at build time via their <code>Cargo.toml</code></p> |
| <p><code>panic_abort</code> is extremely simple: its implementation of <code>__rust_start_panic</code> just aborts, |
| as you would expect.</p> |
| <p><code>panic_unwind</code> is the more interesting case.</p> |
| <p>In its implementation of <code>__rust_start_panic</code>, we take the <code>usize</code>, convert |
| it back to a <code>*mut &mut dyn PanicPayload</code>, dereference it, and call <code>take_box</code> |
| on the <code>&mut dyn PanicPayload</code>. At this point, we have a raw pointer to the payload |
| itself (a <code>*mut (dyn Send + Any)</code>): that is, a raw pointer to the actual value |
| provided by the user who called <code>panic!</code>.</p> |
| <p>At this point, the platform-independent code ends. We now call into |
| platform-specific unwinding logic (e.g <code>unwind</code>). This code is |
| responsible for unwinding the stack, running any 'landing pads' associated |
| with each frame (currently, running destructors), and transferring control |
| to the <code>catch_unwind</code> frame.</p> |
| <p>Note that all panics either abort the process or get caught by some call to <code>catch_unwind</code>. |
| In particular, in std's <a href="https://github.com/rust-lang/rust/blob/HEAD/library/std/src/rt.rs">runtime service</a>, |
| the call to the user-provided <code>main</code> function is wrapped in <code>catch_unwind</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="ast-validation"><a class="header" href="#ast-validation">AST validation</a></h1> |
| <p><em>AST validation</em> is a separate AST pass that visits each |
| item in the tree and performs simple checks. This pass |
| doesn't perform any complex analysis, type checking or |
| name resolution.</p> |
| <p>Before performing any validation, the compiler first expands |
| the macros. Then this pass performs validations to check |
| that each AST item is in the correct state. And when this pass |
| is done, the compiler runs the crate resolution pass.</p> |
| <h2 id="validations"><a class="header" href="#validations">Validations</a></h2> |
| <p>Validations are defined in <code>AstValidator</code> type, which |
| itself is located in <code>rustc_ast_passes</code> crate. This |
| type implements various simple checks which emit errors |
| when certain language rules are broken.</p> |
| <p>In addition, <code>AstValidator</code> implements <code>Visitor</code> trait |
| that defines how to visit AST items (which can be functions, |
| traits, enums, etc).</p> |
| <p>For each item, visitor performs specific checks. For |
| example, when visiting a function declaration, |
| <code>AstValidator</code> checks that the function has:</p> |
| <ul> |
| <li>no more than <code>u16::MAX</code> parameters;</li> |
| <li>c-variadic argument goes the last in the declaration;</li> |
| <li>documentation comments aren't applied to function parameters;</li> |
| <li>and other validations.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="feature-gate-checking"><a class="header" href="#feature-gate-checking">Feature Gate Checking</a></h1> |
| <p><strong>TODO</strong>: this chapter <a href="https://github.com/rust-lang/rustc-dev-guide/issues/1158">#1158</a></p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="lang-items"><a class="header" href="#lang-items">Lang items</a></h1> |
| <p>The compiler has certain pluggable operations; that is, functionality that isn't hard-coded into |
| the language, but is implemented in libraries, with a special marker to tell the compiler it |
| exists. The marker is the attribute <code>#[lang = "..."]</code>, and there are various different values of |
| <code>...</code>, i.e. various different 'lang items'.</p> |
| <p>Many such lang items can be implemented only in one sensible way, such as <code>add</code> (<code>trait core::ops::Add</code>) or <code>future_trait</code> (<code>trait core::future::Future</code>). Others can be overridden to |
| achieve some specific goals; for example, you can control your binary's entrypoint.</p> |
| <p>Features provided by lang items include:</p> |
| <ul> |
| <li>overloadable operators via traits: the traits corresponding to the |
| <code>==</code>, <code><</code>, dereference (<code>*</code>), <code>+</code>, etc. operators are all |
| marked with lang items; those specific four are <code>eq</code>, <code>ord</code>, |
| <code>deref</code>, and <code>add</code> respectively.</li> |
| <li>panicking and stack unwinding; the <code>eh_personality</code>, <code>panic</code> and |
| <code>panic_bounds_checks</code> lang items.</li> |
| <li>the traits in <code>std::marker</code> used to indicate properties of types used by the compiler; |
| lang items <code>send</code>, <code>sync</code> and <code>copy</code>.</li> |
| <li>the special marker types used for variance indicators found in |
| <code>core::marker</code>; lang item <code>phantom_data</code>.</li> |
| </ul> |
| <p>Lang items are loaded lazily by the compiler; e.g. if one never uses <code>Box</code> |
| then there is no need to define functions for <code>exchange_malloc</code> and |
| <code>box_free</code>. <code>rustc</code> will emit an error when an item is needed but not found |
| in the current crate or any that it depends on.</p> |
| <p>Most lang items are defined by the <code>core</code> library, but if you're trying to build an |
| executable with <code>#![no_std]</code>, you'll still need to define a few lang items that are |
| usually provided by <code>std</code>.</p> |
| <h2 id="retrieving-a-language-item"><a class="header" href="#retrieving-a-language-item">Retrieving a language item</a></h2> |
| <p>You can retrieve lang items by calling <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.lang_items"><code>tcx.lang_items()</code></a>.</p> |
| <p>Here's a small example of retrieving the <code>trait Sized {}</code> language item:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// Note that in case of `#![no_core]`, the trait is not available. |
| if let Some(sized_trait_def_id) = tcx.lang_items().sized_trait() { |
| // do something with `sized_trait_def_id` |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Note that <code>sized_trait()</code> returns an <code>Option</code>, not the <code>DefId</code> itself. |
| That's because language items are defined in the standard library, so if someone compiles with |
| <code>#![no_core]</code> (or for some lang items, <code>#![no_std]</code>), the lang item may not be present. |
| You can either:</p> |
| <ul> |
| <li>Give a hard error if the lang item is necessary to continue (don't panic, since this can happen in |
| user code).</li> |
| <li>Proceed with limited functionality, by just omitting whatever you were going to do with the |
| <code>DefId</code>.</li> |
| </ul> |
| <h2 id="list-of-all-language-items"><a class="header" href="#list-of-all-language-items">List of all language items</a></h2> |
| <p>You can find language items in the following places:</p> |
| <ul> |
| <li>An exhaustive reference in the compiler documentation: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/lang_items/enum.LangItem.html"><code>rustc_hir::LangItem</code></a></li> |
| <li>An auto-generated list with source locations by using ripgrep: <code>rg '#\[.*lang =' library/</code></li> |
| </ul> |
| <p>Note that language items are explicitly unstable and may change in any new release.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-hir"><a class="header" href="#the-hir">The HIR</a></h1> |
| <p>The HIR – "High-Level Intermediate Representation" – is the primary IR used |
| in most of rustc. It is a compiler-friendly representation of the abstract |
| syntax tree (AST) that is generated after parsing, macro expansion, and name |
| resolution (see <a href="./hir/lowering.html">Lowering</a> for how the HIR is created). |
| Many parts of HIR resemble Rust surface syntax quite closely, with |
| the exception that some of Rust's expression forms have been desugared away. |
| For example, <code>for</code> loops are converted into a <code>loop</code> and do not appear in |
| the HIR. This makes HIR more amenable to analysis than a normal AST.</p> |
| <p>This chapter covers the main concepts of the HIR.</p> |
| <p>You can view the HIR representation of your code by passing the |
| <code>-Z unpretty=hir-tree</code> flag to rustc:</p> |
| <pre><code class="language-bash">cargo rustc -- -Z unpretty=hir-tree |
| </code></pre> |
| <p>You can also use the <code>-Z unpretty=hir</code> option to generate a HIR |
| that is closer to the original source code expression:</p> |
| <pre><code class="language-bash">cargo rustc -- -Z unpretty=hir |
| </code></pre> |
| <h2 id="out-of-band-storage-and-the-crate-type"><a class="header" href="#out-of-band-storage-and-the-crate-type">Out-of-band storage and the <code>Crate</code> type</a></h2> |
| <p>The top-level data-structure in the HIR is the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Crate.html"><code>Crate</code></a>, which stores |
| the contents of the crate currently being compiled (we only ever |
| construct HIR for the current crate). Whereas in the AST the crate |
| data structure basically just contains the root module, the HIR |
| <code>Crate</code> structure contains a number of maps and other things that |
| serve to organize the content of the crate for easier access.</p> |
| <p>For example, the contents of individual items (e.g. modules, |
| functions, traits, impls, etc) in the HIR are not immediately |
| accessible in the parents. So, for example, if there is a module item |
| <code>foo</code> containing a function <code>bar()</code>:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>mod foo { |
| fn bar() { } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>then in the HIR the representation of module <code>foo</code> (the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Mod.html"><code>Mod</code></a> |
| struct) would only have the <strong><code>ItemId</code></strong> <code>I</code> of <code>bar()</code>. To get the |
| details of the function <code>bar()</code>, we would lookup <code>I</code> in the |
| <code>items</code> map.</p> |
| <p>One nice result from this representation is that one can iterate |
| over all items in the crate by iterating over the key-value pairs |
| in these maps (without the need to trawl through the whole HIR). |
| There are similar maps for things like trait items and impl items, |
| as well as "bodies" (explained below).</p> |
| <p>The other reason to set up the representation this way is for better |
| integration with incremental compilation. This way, if you gain access |
| to an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Item.html"><code>&rustc_hir::Item</code></a> (e.g. for the mod <code>foo</code>), you do not immediately |
| gain access to the contents of the function <code>bar()</code>. Instead, you only |
| gain access to the <strong>id</strong> for <code>bar()</code>, and you must invoke some |
| function to lookup the contents of <code>bar()</code> given its id; this gives |
| the compiler a chance to observe that you accessed the data for |
| <code>bar()</code>, and then record the dependency.</p> |
| <p><a id="hir-id"></a></p> |
| <h2 id="identifiers-in-the-hir"><a class="header" href="#identifiers-in-the-hir">Identifiers in the HIR</a></h2> |
| <p>The HIR uses a bunch of different identifiers that coexist and serve different purposes.</p> |
| <ul> |
| <li> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a>, as the name suggests, identifies a particular definition, or top-level |
| item, in a given crate. It is composed of two parts: a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.CrateNum.html"><code>CrateNum</code></a> which identifies |
| the crate the definition comes from, and a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefIndex.html"><code>DefIndex</code></a> which identifies the definition |
| within the crate. Unlike <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html"><code>HirId</code></a>s, there isn't a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a> for every expression, which |
| makes them more stable across compilations.</p> |
| </li> |
| <li> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.LocalDefId.html"><code>LocalDefId</code></a> is basically a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a> that is known to come from the current crate. |
| This allows us to drop the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.CrateNum.html"><code>CrateNum</code></a> part, and use the type system to ensure that |
| only local definitions are passed to functions that expect a local definition.</p> |
| </li> |
| <li> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html"><code>HirId</code></a> uniquely identifies a node in the HIR of the current crate. It is composed |
| of two parts: an <code>owner</code> and a <code>local_id</code> that is unique within the <code>owner</code>. This |
| combination makes for more stable values which are helpful for incremental compilation. |
| Unlike <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a>s, a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html"><code>HirId</code></a> can refer to <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.Node.html">fine-grained entities</a> like expressions, |
| but stays local to the current crate.</p> |
| </li> |
| <li> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.BodyId.html"><code>BodyId</code></a> identifies a HIR <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Body.html"><code>Body</code></a> in the current crate. It is currently only |
| a wrapper around a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html"><code>HirId</code></a>. For more info about HIR bodies, please refer to the |
| <a href="./hir.html#hir-bodies">HIR chapter</a>.</p> |
| </li> |
| </ul> |
| <p>These identifiers can be converted into one another through the <code>TyCtxt</code>.</p> |
| <h2 id="hir-operations"><a class="header" href="#hir-operations">HIR Operations</a></h2> |
| <p>Most of the time when you are working with the HIR, you will do so via |
| <code>TyCtxt</code>. It contains a number of methods, defined in the <code>hir::map</code> module and |
| mostly prefixed with <code>hir_</code>, to convert between IDs of various kinds and to |
| lookup data associated with a HIR node.</p> |
| <p>For example, if you have a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.LocalDefId.html"><code>LocalDefId</code></a>, and you would like to convert it |
| to a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html"><code>HirId</code></a>, you can use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.local_def_id_to_hir_id"><code>tcx.local_def_id_to_hir_id(def_id)</code></a>. |
| You need a <code>LocalDefId</code>, rather than a <code>DefId</code>, since only local items have HIR nodes.</p> |
| <p>Similarly, you can use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir_node"><code>tcx.hir_node(n)</code></a> to lookup the node for a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html"><code>HirId</code></a>. This returns a <code>Option<Node<'hir>></code>, where <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.Node.html"><code>Node</code></a> is an enum |
| defined in the map. By matching on this, you can find out what sort of |
| node the <code>HirId</code> referred to and also get a pointer to the data |
| itself. Often, you know what sort of node <code>n</code> is – e.g. if you know |
| that <code>n</code> must be some HIR expression, you can do |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.expect_expr"><code>tcx.hir_expect_expr(n)</code></a>, which will extract and return the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Expr.html"><code>&hir::Expr</code></a>, panicking if <code>n</code> is not in fact an expression.</p> |
| <p>Finally, you can find the parents of nodes, via |
| calls like <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.parent_hir_node"><code>tcx.parent_hir_node(n)</code></a>.</p> |
| <h2 id="hir-bodies"><a class="header" href="#hir-bodies">HIR Bodies</a></h2> |
| <p>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Body.html"><code>rustc_hir::Body</code></a> represents some kind of executable code, such as the body |
| of a function/closure or the definition of a constant. Bodies are |
| associated with an <strong>owner</strong>, which is typically some kind of item |
| (e.g. an <code>fn()</code> or <code>const</code>), but could also be a closure expression |
| (e.g. <code>|x, y| x + y</code>). You can use the <code>TyCtxt</code> to find the body |
| associated with a given def-id (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir_maybe_body_owned_by"><code>hir_maybe_body_owned_by</code></a>) or to find |
| the owner of a body (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir_body_owner_def_id"><code>hir_body_owner_def_id</code></a>).</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="ast-lowering-1"><a class="header" href="#ast-lowering-1">AST lowering</a></h1> |
| <p>The AST lowering step converts AST to <a href="hir/../hir.html">HIR</a>. |
| This means many structures are removed if they are irrelevant |
| for type analysis or similar syntax agnostic analyses. Examples |
| of such structures include but are not limited to</p> |
| <ul> |
| <li>Parenthesis |
| <ul> |
| <li>Removed without replacement, the tree structure makes order explicit</li> |
| </ul> |
| </li> |
| <li><code>for</code> loops |
| <ul> |
| <li>Converted to <code>match</code> + <code>loop</code> + <code>match</code></li> |
| </ul> |
| </li> |
| <li>Universal <code>impl Trait</code> |
| <ul> |
| <li>Converted to generic arguments |
| (but with some flags, to know that the user didn't write them)</li> |
| </ul> |
| </li> |
| <li>Existential <code>impl Trait</code> |
| <ul> |
| <li>Converted to a virtual <code>existential type</code> declaration</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Lowering needs to uphold several invariants in order to not trigger the |
| sanity checks in <code>compiler/rustc_passes/src/hir_id_validator.rs</code>:</p> |
| <ol> |
| <li>A <code>HirId</code> must be used if created. So if you use the <code>lower_node_id</code>, |
| you <em>must</em> use the resulting <code>NodeId</code> or <code>HirId</code> (either is fine, since |
| any <code>NodeId</code>s in the <code>HIR</code> are checked for existing <code>HirId</code>s)</li> |
| <li>Lowering a <code>HirId</code> must be done in the scope of the <em>owning</em> item. |
| This means you need to use <code>with_hir_id_owner</code> if you are creating parts |
| of an item other than the one being currently lowered. This happens for |
| example during the lowering of existential <code>impl Trait</code></li> |
| <li>A <code>NodeId</code> that will be placed into a HIR structure must be lowered, |
| even if its <code>HirId</code> is unused. Calling |
| <code>let _ = self.lower_node_id(node_id);</code> is perfectly legitimate.</li> |
| <li>If you are creating new nodes that didn't exist in the <code>AST</code>, you <em>must</em> |
| create new ids for them. This is done by calling the <code>next_id</code> method, |
| which produces both a new <code>NodeId</code> as well as automatically lowering it |
| for you so you also get the <code>HirId</code>.</li> |
| </ol> |
| <p>If you are creating new <code>DefId</code>s, since each <code>DefId</code> needs to have a |
| corresponding <code>NodeId</code>, it is advisable to add these <code>NodeId</code>s to the |
| <code>AST</code> so you don't have to generate new ones during lowering. This has |
| the advantage of creating a way to find the <code>DefId</code> of something via its |
| <code>NodeId</code>. If lowering needs this <code>DefId</code> in multiple places, you can't |
| generate a new <code>NodeId</code> in all those places because you'd also get a new |
| <code>DefId</code> then. With a <code>NodeId</code> from the <code>AST</code> this is not an issue.</p> |
| <p>Having the <code>NodeId</code> also allows the <code>DefCollector</code> to generate the <code>DefId</code>s |
| instead of lowering having to do it on the fly. Centralizing the <code>DefId</code> |
| generation in one place makes it easier to refactor and reason about.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="ambigunambig-types-and-consts"><a class="header" href="#ambigunambig-types-and-consts">Ambig/Unambig Types and Consts</a></h1> |
| <p>Types and Consts args in the HIR can be in two kinds of positions ambiguous (ambig) or unambiguous (unambig). Ambig positions are where |
| it would be valid to parse either a type or a const, unambig positions are where only one kind would be valid to |
| parse.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn func<T, const N: usize>(arg: T) { |
| // ^ Unambig type position |
| let a: _ = arg; |
| // ^ Unambig type position |
| |
| func::<T, N>(arg); |
| // ^ ^ |
| // ^^^^ Ambig position |
| |
| let _: [u8; 10]; |
| // ^^ ^^ Unambig const position |
| // ^^ Unambig type position |
| } |
| |
| <span class="boring">}</span></code></pre></pre> |
| <p>Most types/consts in ambig positions are able to be disambiguated as either a type or const during parsing. Single segment paths are always represented as types in the AST but may get resolved to a const parameter during name resolution, then lowered to a const argument during ast-lowering. The only generic arguments which remain ambiguous after lowering are inferred generic arguments (<code>_</code>) in path segments. For example, in <code>Foo<_></code> it is not clear whether the <code>_</code> argument is an inferred type argument, or an inferred const argument.</p> |
| <p>In unambig positions, inferred arguments are represented with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.TyKind.html#variant.Infer"><code>hir::TyKind::Infer</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.ConstArgKind.html#variant.Infer"><code>hir::ConstArgKind::Infer</code></a> depending on whether it is a type or const position respectively. |
| In ambig positions, inferred arguments are represented with <code>hir::GenericArg::Infer</code>.</p> |
| <p>A naive implementation of this would result in there being potentially 5 places where you might think an inferred type/const could be found in the HIR from looking at the structure of the HIR:</p> |
| <ol> |
| <li>In unambig type position as a <code>hir::TyKind::Infer</code></li> |
| <li>In unambig const arg position as a <code>hir::ConstArgKind::Infer</code></li> |
| <li>In an ambig position as a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.GenericArg.html#variant.Type"><code>GenericArg::Type(TyKind::Infer)</code></a></li> |
| <li>In an ambig position as a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.GenericArg.html#variant.Const"><code>GenericArg::Const(ConstArgKind::Infer)</code></a></li> |
| <li>In an ambig position as a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.GenericArg.html#variant.Infer"><code>GenericArg::Infer</code></a></li> |
| </ol> |
| <p>Note that places 3 and 4 would never actually be possible to encounter as we always lower to <code>GenericArg::Infer</code> in generic arg position.</p> |
| <p>This has a few failure modes:</p> |
| <ul> |
| <li>People may write visitors which check for <code>GenericArg::Infer</code> but forget to check for <code>hir::TyKind/ConstArgKind::Infer</code>, only handling infers in ambig positions by accident.</li> |
| <li>People may write visitors which check for <code>hir::TyKind/ConstArgKind::Infer</code> but forget to check for <code>GenericArg::Infer</code>, only handling infers in unambig positions by accident.</li> |
| <li>People may write visitors which check for <code>GenericArg::Type/Const(TyKind/ConstArgKind::Infer)</code> and <code>GenericArg::Infer</code>, not realising that we never represent inferred types/consts in ambig positions as a <code>GenericArg::Type/Const</code>.</li> |
| <li>People may write visitors which check for <em>only</em> <code>TyKind::Infer</code> and not <code>ConstArgKind::Infer</code> forgetting that there are also inferred const arguments (and vice versa).</li> |
| </ul> |
| <p>To make writing HIR visitors less error prone when caring about inferred types/consts we have a relatively complex system:</p> |
| <ol> |
| <li> |
| <p>We have different types in the compiler for when a type or const is in an unambig or ambig position, <code>hir::Ty<AmbigArg></code> and <code>hir::Ty<()></code>. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.AmbigArg.html"><code>AmbigArg</code></a> is an uninhabited type which we use in the <code>Infer</code> variant of <code>TyKind</code> and <code>ConstArgKind</code> to selectively "disable" it if we are in an ambig position.</p> |
| </li> |
| <li> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/intravisit/trait.Visitor.html#method.visit_ty"><code>visit_ty</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/intravisit/trait.Visitor.html#method.visit_const_arg"><code>visit_const_arg</code></a> methods on HIR visitors only accept the ambig position versions of types/consts. Unambig types/consts are implicitly converted to ambig types/consts during the visiting process, with the <code>Infer</code> variant handled by a dedicated <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/intravisit/trait.Visitor.html#method.visit_infer"><code>visit_infer</code></a> method.</p> |
| </li> |
| </ol> |
| <p>This has a number of benefits:</p> |
| <ul> |
| <li>It's clear that <code>GenericArg::Type/Const</code> cannot represent inferred type/const arguments</li> |
| <li>Implementors of <code>visit_ty</code> and <code>visit_const_arg</code> will never encounter inferred types/consts making it impossible to write a visitor that seems to work right but handles edge cases wrong</li> |
| <li>The <code>visit_infer</code> method handles <em>all</em> cases of inferred type/consts in the HIR making it easy for visitors to handle inferred type/consts in one dedicated place and not forget cases</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="hir-debugging"><a class="header" href="#hir-debugging">HIR Debugging</a></h1> |
| <p>Use the <code>-Z unpretty=hir</code> flag to produce a human-readable representation of the HIR. |
| For cargo projects this can be done with <code>cargo rustc -- -Z unpretty=hir</code>. |
| This output is useful when you need to see at a glance how your code was desugared and transformed |
| during AST lowering.</p> |
| <p>For a full <code>Debug</code> dump of the data in the HIR, use the <code>-Z unpretty=hir-tree</code> flag. |
| This may be useful when you need to see the full structure of the HIR from the perspective of the |
| compiler.</p> |
| <p>If you are trying to correlate <code>NodeId</code>s or <code>DefId</code>s with source code, the |
| <code>-Z unpretty=expanded,identified</code> flag may be useful.</p> |
| <p>TODO: anything else? <a href="https://github.com/rust-lang/rustc-dev-guide/issues/1159">#1159</a></p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-thir"><a class="header" href="#the-thir">The THIR</a></h1> |
| <p>The THIR ("Typed High-Level Intermediate Representation"), previously called HAIR for |
| "High-Level Abstract IR", is another IR used by rustc that is generated after |
| <a href="./type-checking.html">type checking</a>. It is (as of <!-- date-check --> January 2024) used for |
| <a href="./mir/construction.html">MIR construction</a>, <a href="./pat-exhaustive-checking.html">exhaustiveness checking</a>, and <a href="./unsafety-checking.html">unsafety checking</a>.</p> |
| <p>As the name might suggest, the THIR is a lowered version of the <a href="./hir.html">HIR</a> where all |
| the types have been filled in, which is possible after type checking has completed. |
| But it has some other interesting features that distinguish it from the HIR:</p> |
| <ul> |
| <li> |
| <p>Like the MIR, the THIR only represents bodies, i.e. "executable code"; this includes |
| function bodies, but also <code>const</code> initializers, for example. Specifically, all <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.BodyOwnerKind.html">body owners</a> have |
| THIR created. Consequently, the THIR has no representation for items like <code>struct</code>s or <code>trait</code>s.</p> |
| </li> |
| <li> |
| <p>Each body of THIR is only stored temporarily and is dropped as soon as it's no longer |
| needed, as opposed to being stored until the end of the compilation process (which |
| is what is done with the HIR).</p> |
| </li> |
| <li> |
| <p>Besides making the types of all nodes available, the THIR also has additional |
| desugaring compared to the HIR. For example, automatic references and dereferences |
| are made explicit, and method calls and overloaded operators are converted into |
| plain function calls. Destruction scopes are also made explicit.</p> |
| </li> |
| <li> |
| <p>Statements, expressions, and match arms are stored separately. For example, statements in the |
| <code>stmts</code> array reference expressions by their index (represented as a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/thir/struct.ExprId.html"><code>ExprId</code></a>) in the <code>exprs</code> |
| array.</p> |
| </li> |
| </ul> |
| <p>The THIR lives in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/index.html"><code>rustc_mir_build::thir</code></a>. To construct a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/thir/struct.Expr.html"><code>thir::Expr</code></a>, |
| you can use the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.thir_body"><code>thir_body</code></a> function, passing in the memory arena where the THIR |
| will be allocated. Dropping this arena will result in the THIR being destroyed, |
| which is useful to keep peak memory in check. Having a THIR representation of |
| all bodies of a crate in memory at the same time would be very heavy.</p> |
| <p>You can get a debug representation of the THIR by passing the <code>-Zunpretty=thir-tree</code> flag |
| to <code>rustc</code>.</p> |
| <p>To demonstrate, let's use the following example:</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let x = 1 + 2; |
| }</code></pre></pre> |
| <p>Here is how that gets represented in THIR (as of <!-- date-check --> Aug 2022):</p> |
| <pre><pre class="playground"><code class="language-rust no_run"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>Thir { |
| // no match arms |
| arms: [], |
| exprs: [ |
| // expression 0, a literal with a value of 1 |
| Expr { |
| ty: i32, |
| temp_lifetime: Some( |
| Node(1), |
| ), |
| span: oneplustwo.rs:2:13: 2:14 (#0), |
| kind: Literal { |
| lit: Spanned { |
| node: Int( |
| 1, |
| Unsuffixed, |
| ), |
| span: oneplustwo.rs:2:13: 2:14 (#0), |
| }, |
| neg: false, |
| }, |
| }, |
| // expression 1, scope surrounding literal 1 |
| Expr { |
| ty: i32, |
| temp_lifetime: Some( |
| Node(1), |
| ), |
| span: oneplustwo.rs:2:13: 2:14 (#0), |
| kind: Scope { |
| // reference to expression 0 above |
| region_scope: Node(3), |
| lint_level: Explicit( |
| HirId { |
| owner: DefId(0:3 ~ oneplustwo[6932]::main), |
| local_id: 3, |
| }, |
| ), |
| value: e0, |
| }, |
| }, |
| // expression 2, literal 2 |
| Expr { |
| ty: i32, |
| temp_lifetime: Some( |
| Node(1), |
| ), |
| span: oneplustwo.rs:2:17: 2:18 (#0), |
| kind: Literal { |
| lit: Spanned { |
| node: Int( |
| 2, |
| Unsuffixed, |
| ), |
| span: oneplustwo.rs:2:17: 2:18 (#0), |
| }, |
| neg: false, |
| }, |
| }, |
| // expression 3, scope surrounding literal 2 |
| Expr { |
| ty: i32, |
| temp_lifetime: Some( |
| Node(1), |
| ), |
| span: oneplustwo.rs:2:17: 2:18 (#0), |
| kind: Scope { |
| region_scope: Node(4), |
| lint_level: Explicit( |
| HirId { |
| owner: DefId(0:3 ~ oneplustwo[6932]::main), |
| local_id: 4, |
| }, |
| ), |
| // reference to expression 2 above |
| value: e2, |
| }, |
| }, |
| // expression 4, represents 1 + 2 |
| Expr { |
| ty: i32, |
| temp_lifetime: Some( |
| Node(1), |
| ), |
| span: oneplustwo.rs:2:13: 2:18 (#0), |
| kind: Binary { |
| op: Add, |
| // references to scopes surrounding literals above |
| lhs: e1, |
| rhs: e3, |
| }, |
| }, |
| // expression 5, scope surrounding expression 4 |
| Expr { |
| ty: i32, |
| temp_lifetime: Some( |
| Node(1), |
| ), |
| span: oneplustwo.rs:2:13: 2:18 (#0), |
| kind: Scope { |
| region_scope: Node(5), |
| lint_level: Explicit( |
| HirId { |
| owner: DefId(0:3 ~ oneplustwo[6932]::main), |
| local_id: 5, |
| }, |
| ), |
| value: e4, |
| }, |
| }, |
| // expression 6, block around statement |
| Expr { |
| ty: (), |
| temp_lifetime: Some( |
| Node(9), |
| ), |
| span: oneplustwo.rs:1:11: 3:2 (#0), |
| kind: Block { |
| body: Block { |
| targeted_by_break: false, |
| region_scope: Node(8), |
| opt_destruction_scope: None, |
| span: oneplustwo.rs:1:11: 3:2 (#0), |
| // reference to statement 0 below |
| stmts: [ |
| s0, |
| ], |
| expr: None, |
| safety_mode: Safe, |
| }, |
| }, |
| }, |
| // expression 7, scope around block in expression 6 |
| Expr { |
| ty: (), |
| temp_lifetime: Some( |
| Node(9), |
| ), |
| span: oneplustwo.rs:1:11: 3:2 (#0), |
| kind: Scope { |
| region_scope: Node(9), |
| lint_level: Explicit( |
| HirId { |
| owner: DefId(0:3 ~ oneplustwo[6932]::main), |
| local_id: 9, |
| }, |
| ), |
| value: e6, |
| }, |
| }, |
| // destruction scope around expression 7 |
| Expr { |
| ty: (), |
| temp_lifetime: Some( |
| Node(9), |
| ), |
| span: oneplustwo.rs:1:11: 3:2 (#0), |
| kind: Scope { |
| region_scope: Destruction(9), |
| lint_level: Inherited, |
| value: e7, |
| }, |
| }, |
| ], |
| stmts: [ |
| // let statement |
| Stmt { |
| kind: Let { |
| remainder_scope: Remainder { block: 8, first_statement_index: 0}, |
| init_scope: Node(1), |
| pattern: Pat { |
| ty: i32, |
| span: oneplustwo.rs:2:9: 2:10 (#0), |
| kind: Binding { |
| mutability: Not, |
| name: "x", |
| mode: ByValue, |
| var: LocalVarId( |
| HirId { |
| owner: DefId(0:3 ~ oneplustwo[6932]::main), |
| local_id: 7, |
| }, |
| ), |
| ty: i32, |
| subpattern: None, |
| is_primary: true, |
| }, |
| }, |
| initializer: Some( |
| e5, |
| ), |
| else_block: None, |
| lint_level: Explicit( |
| HirId { |
| owner: DefId(0:3 ~ oneplustwo[6932]::main), |
| local_id: 6, |
| }, |
| ), |
| }, |
| opt_destruction_scope: Some( |
| Destruction(1), |
| ), |
| }, |
| ], |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-mir-mid-level-ir"><a class="header" href="#the-mir-mid-level-ir">The MIR (Mid-level IR)</a></h1> |
| <p>MIR is Rust's <em>Mid-level Intermediate Representation</em>. It is |
| constructed from <a href="mir/../hir.html">HIR</a>. MIR was introduced in |
| <a href="https://rust-lang.github.io/rfcs/1211-mir.html">RFC 1211</a>. It is a radically simplified form of Rust that is used for |
| certain flow-sensitive safety checks – notably the borrow checker! – |
| and also for optimization and code generation.</p> |
| <p>If you'd like a very high-level introduction to MIR, as well as some |
| of the compiler concepts that it relies on (such as control-flow |
| graphs and desugaring), you may enjoy the |
| <a href="https://blog.rust-lang.org/2016/04/19/MIR.html">rust-lang blog post that introduced MIR</a>.</p> |
| <h2 id="introduction-to-mir"><a class="header" href="#introduction-to-mir">Introduction to MIR</a></h2> |
| <p>MIR is defined in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/index.html"><code>compiler/rustc_middle/src/mir/</code></a> module, but much of the code |
| that manipulates it is found in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/index.html"><code>compiler/rustc_mir_build</code></a>, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/index.html"><code>compiler/rustc_mir_transform</code></a>, and |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/index.html"><code>compiler/rustc_mir_dataflow</code></a>.</p> |
| <p>Some of the key characteristics of MIR are:</p> |
| <ul> |
| <li>It is based on a <a href="mir/../appendix/background.html#cfg">control-flow graph</a>.</li> |
| <li>It does not have nested expressions.</li> |
| <li>All types in MIR are fully explicit.</li> |
| </ul> |
| <h2 id="key-mir-vocabulary"><a class="header" href="#key-mir-vocabulary">Key MIR vocabulary</a></h2> |
| <p>This section introduces the key concepts of MIR, summarized here:</p> |
| <ul> |
| <li><strong>Basic blocks</strong>: units of the control-flow graph, consisting of: |
| <ul> |
| <li><strong>statements:</strong> actions with one successor</li> |
| <li><strong>terminators:</strong> actions with potentially multiple successors; always at |
| the end of a block</li> |
| <li>(if you're not familiar with the term <em>basic block</em>, see the <a href="mir/../appendix/background.html#cfg">background |
| chapter</a>)</li> |
| </ul> |
| </li> |
| <li><strong>Locals:</strong> Memory locations allocated on the stack (conceptually, at |
| least), such as function arguments, local variables, and |
| temporaries. These are identified by an index, written with a |
| leading underscore, like <code>_1</code>. There is also a special "local" |
| (<code>_0</code>) allocated to store the return value.</li> |
| <li><strong>Places:</strong> expressions that identify a location in memory, like <code>_1</code> or |
| <code>_1.f</code>.</li> |
| <li><strong>Rvalues:</strong> expressions that produce a value. The "R" stands for |
| the fact that these are the "right-hand side" of an assignment. |
| <ul> |
| <li><strong>Operands:</strong> the arguments to an rvalue, which can either be a |
| constant (like <code>22</code>) or a place (like <code>_1</code>).</li> |
| </ul> |
| </li> |
| </ul> |
| <p>You can get a feeling for how MIR is constructed by translating simple |
| programs into MIR and reading the pretty printed output. In fact, the |
| playground makes this easy, since it supplies a MIR button that will |
| show you the MIR for your program. Try putting this program into play |
| (or <a href="https://play.rust-lang.org/?gist=30074856e62e74e91f06abd19bd72ece&version=stable&edition=2021">clicking on this link</a>), and then clicking the "MIR" |
| button on the top:</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let mut vec = Vec::new(); |
| vec.push(1); |
| vec.push(2); |
| }</code></pre></pre> |
| <p>You should see something like:</p> |
| <pre><code class="language-mir">// WARNING: This output format is intended for human consumers only |
| // and is subject to change without notice. Knock yourself out. |
| fn main() -> () { |
| ... |
| } |
| </code></pre> |
| <p>This is the MIR format for the <code>main</code> function. |
| MIR shown by above link is optimized. |
| Some statements like <code>StorageLive</code> are removed in optimization. |
| This happens because the compiler notices the value is never accessed in the code. |
| We can use <code>rustc [filename].rs -Z mir-opt-level=0 --emit mir</code> to view unoptimized MIR. |
| This requires the nightly toolchain.</p> |
| <p><strong>Variable declarations.</strong> If we drill in a bit, we'll see it begins |
| with a bunch of variable declarations. They look like this:</p> |
| <pre><code class="language-mir">let mut _0: (); // return place |
| let mut _1: std::vec::Vec<i32>; // in scope 0 at src/main.rs:2:9: 2:16 |
| let mut _2: (); |
| let mut _3: &mut std::vec::Vec<i32>; |
| let mut _4: (); |
| let mut _5: &mut std::vec::Vec<i32>; |
| </code></pre> |
| <p>You can see that variables in MIR don't have names, they have indices, |
| like <code>_0</code> or <code>_1</code>. We also intermingle the user's variables (e.g., |
| <code>_1</code>) with temporary values (e.g., <code>_2</code> or <code>_3</code>). You can tell apart |
| user-defined variables because they have debuginfo associated to them (see below).</p> |
| <p><strong>User variable debuginfo.</strong> Below the variable declarations, we find the only |
| hint that <code>_1</code> represents a user variable:</p> |
| <pre><code class="language-mir">scope 1 { |
| debug vec => _1; // in scope 1 at src/main.rs:2:9: 2:16 |
| } |
| </code></pre> |
| <p>Each <code>debug <Name> => <Place>;</code> annotation describes a named user variable, |
| and where (i.e. the place) a debugger can find the data of that variable. |
| Here the mapping is trivial, but optimizations may complicate the place, |
| or lead to multiple user variables sharing the same place. |
| Additionally, closure captures are described using the same system, and so |
| they're complicated even without optimizations, e.g.: <code>debug x => (*((*_1).0: &T));</code>.</p> |
| <p>The "scope" blocks (e.g., <code>scope 1 { .. }</code>) describe the lexical structure of |
| the source program (which names were in scope when), so any part of the program |
| annotated with <code>// in scope 0</code> would be missing <code>vec</code>, if you were stepping |
| through the code in a debugger, for example.</p> |
| <p><strong>Basic blocks.</strong> Reading further, we see our first <strong>basic block</strong> (naturally |
| it may look slightly different when you view it, and I am ignoring some of the |
| comments):</p> |
| <pre><code class="language-mir">bb0: { |
| StorageLive(_1); |
| _1 = const <std::vec::Vec<T>>::new() -> bb2; |
| } |
| </code></pre> |
| <p>A basic block is defined by a series of <strong>statements</strong> and a final |
| <strong>terminator</strong>. In this case, there is one statement:</p> |
| <pre><code class="language-mir">StorageLive(_1); |
| </code></pre> |
| <p>This statement indicates that the variable <code>_1</code> is "live", meaning |
| that it may be used later – this will persist until we encounter a |
| <code>StorageDead(_1)</code> statement, which indicates that the variable <code>_1</code> is |
| done being used. These "storage statements" are used by LLVM to |
| allocate stack space.</p> |
| <p>The <strong>terminator</strong> of the block <code>bb0</code> is the call to <code>Vec::new</code>:</p> |
| <pre><code class="language-mir">_1 = const <std::vec::Vec<T>>::new() -> bb2; |
| </code></pre> |
| <p>Terminators are different from statements because they can have more |
| than one successor – that is, control may flow to different |
| places. Function calls like the call to <code>Vec::new</code> are always |
| terminators because of the possibility of unwinding, although in the |
| case of <code>Vec::new</code> we are able to see that indeed unwinding is not |
| possible, and hence we list only one successor block, <code>bb2</code>.</p> |
| <p>If we look ahead to <code>bb2</code>, we will see it looks like this:</p> |
| <pre><code class="language-mir">bb2: { |
| StorageLive(_3); |
| _3 = &mut _1; |
| _2 = const <std::vec::Vec<T>>::push(move _3, const 1i32) -> [return: bb3, unwind: bb4]; |
| } |
| </code></pre> |
| <p>Here there are two statements: another <code>StorageLive</code>, introducing the <code>_3</code> |
| temporary, and then an assignment:</p> |
| <pre><code class="language-mir">_3 = &mut _1; |
| </code></pre> |
| <p>Assignments in general have the form:</p> |
| <pre><code class="language-text"><Place> = <Rvalue> |
| </code></pre> |
| <p>A place is an expression like <code>_3</code>, <code>_3.f</code> or <code>*_3</code> – it denotes a |
| location in memory. An <strong>Rvalue</strong> is an expression that creates a |
| value: in this case, the rvalue is a mutable borrow expression, which |
| looks like <code>&mut <Place></code>. So we can kind of define a grammar for |
| rvalues like so:</p> |
| <pre><code class="language-text"><Rvalue> = & (mut)? <Place> |
| | <Operand> + <Operand> |
| | <Operand> - <Operand> |
| | ... |
| |
| <Operand> = Constant |
| | copy Place |
| | move Place |
| </code></pre> |
| <p>As you can see from this grammar, rvalues cannot be nested – they can |
| only reference places and constants. Moreover, when you use a place, |
| we indicate whether we are <strong>copying it</strong> (which requires that the |
| place have a type <code>T</code> where <code>T: Copy</code>) or <strong>moving it</strong> (which works |
| for a place of any type). So, for example, if we had the expression <code>x = a + b + c</code> in Rust, that would get compiled to two statements and a |
| temporary:</p> |
| <pre><code class="language-mir">TMP1 = a + b |
| x = TMP1 + c |
| </code></pre> |
| <p>(<a href="https://play.rust-lang.org/?gist=1751196d63b2a71f8208119e59d8a5b6&version=stable">Try it and see</a>, though you may want to do release mode to skip |
| over the overflow checks.)</p> |
| <h2 id="mir-data-types"><a class="header" href="#mir-data-types">MIR data types</a></h2> |
| <p>The MIR data types are defined in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/index.html"><code>compiler/rustc_middle/src/mir/</code></a> |
| module. Each of the key concepts mentioned in the previous section |
| maps in a fairly straightforward way to a Rust type.</p> |
| <p>The main MIR data type is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Body.html"><code>Body</code></a>. It contains the data for a single |
| function (along with sub-instances of Mir for "promoted constants", |
| but <a href="mir/index.html#promoted">you can read about those below</a>).</p> |
| <ul> |
| <li><strong>Basic blocks</strong>: The basic blocks are stored in the field |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Body.html#structfield.basic_blocks"><code>Body::basic_blocks</code></a>; this is a vector |
| of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.BasicBlockData.html"><code>BasicBlockData</code></a> structures. Nobody ever references a |
| basic block directly: instead, we pass around <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.BasicBlock.html"><code>BasicBlock</code></a> |
| values, which are <a href="mir/../appendix/glossary.html#newtype">newtype'd</a> indices into this vector.</li> |
| <li><strong>Statements</strong> are represented by the type <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Statement.html"><code>Statement</code></a>.</li> |
| <li><strong>Terminators</strong> are represented by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/terminator/struct.Terminator.html"><code>Terminator</code></a>.</li> |
| <li><strong>Locals</strong> are represented by a <a href="mir/../appendix/glossary.html#newtype">newtype'd</a> index type <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Local.html"><code>Local</code></a>. |
| The data for a local variable is found in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Body.html#structfield.local_decls"><code>Body::local_decls</code></a> vector. There is also a special constant |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/constant.RETURN_PLACE.html"><code>RETURN_PLACE</code></a> identifying the special "local" representing the return value.</li> |
| <li><strong>Places</strong> are identified by the struct <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a>. There are a few |
| fields: |
| <ul> |
| <li>Local variables like <code>_1</code></li> |
| <li><strong>Projections</strong>, which are fields or other things that "project |
| out" from a base place. These are represented by the <a href="mir/../appendix/glossary.html#newtype">newtype'd</a> type |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.ProjectionElem.html"><code>ProjectionElem</code></a>. So e.g. the place <code>_1.f</code> is a projection, |
| with <code>f</code> being the "projection element" and <code>_1</code> being the base |
| path. <code>*_1</code> is also a projection, with the <code>*</code> being represented |
| by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.ProjectionElem.html#variant.Deref"><code>ProjectionElem::Deref</code></a> element.</li> |
| </ul> |
| </li> |
| <li><strong>Rvalues</strong> are represented by the enum <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.Rvalue.html"><code>Rvalue</code></a>.</li> |
| <li><strong>Operands</strong> are represented by the enum <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.Operand.html"><code>Operand</code></a>.</li> |
| </ul> |
| <h2 id="representing-constants"><a class="header" href="#representing-constants">Representing constants</a></h2> |
| <p>When code has reached the MIR stage, constants can generally come in two forms: |
| <em>MIR constants</em> (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.Const.html"><code>mir::Constant</code></a>) and <em>type system constants</em> (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Const.html"><code>ty::Const</code></a>). |
| MIR constants are used as operands: in <code>x + CONST</code>, <code>CONST</code> is a MIR constant; |
| similarly, in <code>x + 2</code>, <code>2</code> is a MIR constant. Type system constants are used in |
| the type system, in particular for array lengths but also for const generics.</p> |
| <p>Generally, both kinds of constants can be "unevaluated" or "already evaluated". |
| An unevaluated constant simply stores the <code>DefId</code> of what needs to be evaluated |
| to compute this result. An evaluated constant (a "value") has already been |
| computed; their representation differs between type system constants and MIR |
| constants: MIR constants evaluate to a <code>mir::ConstValue</code>; type system constants |
| evaluate to a <code>ty::ValTree</code>.</p> |
| <p>Type system constants have some more variants to support const generics: they |
| can refer to local const generic parameters, and they are subject to inference. |
| Furthermore, the <code>mir::Constant::Ty</code> variant lets us use an arbitrary type |
| system constant as a MIR constant; this happens whenever a const generic |
| parameter is used as an operand.</p> |
| <h3 id="mir-constant-values"><a class="header" href="#mir-constant-values">MIR constant values</a></h3> |
| <p>In general, a MIR constant value (<code>mir::ConstValue</code>) was computed by evaluating |
| some constant the user wrote. This <a href="mir/../const-eval.html">const evaluation</a> produces |
| a very low-level representation of the result in terms of individual bytes. We |
| call this an "indirect" constant (<code>mir::ConstValue::Indirect</code>) since the value |
| is stored in-memory.</p> |
| <p>However, storing everything in-memory would be awfully inefficient. Hence there |
| are some other variants in <code>mir::ConstValue</code> that can represent certain simple |
| and common values more efficiently. In particular, everything that can be |
| directly written as a literal in Rust (integers, floats, chars, bools, but also |
| <code>"string literals"</code> and <code>b"byte string literals"</code>) has an optimized variant that |
| avoids the full overhead of the in-memory representation.</p> |
| <h3 id="valtrees"><a class="header" href="#valtrees">ValTrees</a></h3> |
| <p>An evaluated type system constant is a "valtree". The <code>ty::ValTree</code> datastructure |
| allows us to represent</p> |
| <ul> |
| <li>arrays,</li> |
| <li>many structs,</li> |
| <li>tuples,</li> |
| <li>enums and,</li> |
| <li>most primitives.</li> |
| </ul> |
| <p>The most important rule for |
| this representation is that every value must be uniquely represented. In other |
| words: a specific value must only be representable in one specific way. For example: there is only |
| one way to represent an array of two integers as a <code>ValTree</code>: |
| <code>Branch([Leaf(first_int), Leaf(second_int)])</code>. |
| Even though theoretically a <code>[u32; 2]</code> could be encoded in a <code>u64</code> and thus just be a |
| <code>Leaf(bits_of_two_u32)</code>, that is not a legal construction of <code>ValTree</code> |
| (and is very complex to do, so it is unlikely anyone is tempted to do so).</p> |
| <p>These rules also mean that some values are not representable. There can be no <code>union</code>s in type |
| level constants, as it is not clear how they should be represented, because their active variant |
| is unknown. Similarly there is no way to represent raw pointers, as addresses are unknown at |
| compile-time and thus we cannot make any assumptions about them. References on the other hand |
| <em>can</em> be represented, as equality for references is defined as equality on their value, so we |
| ignore their address and just look at the backing value. We must make sure that the pointer values |
| of the references are not observable at compile time. We thus encode <code>&42</code> exactly like <code>42</code>. |
| Any conversion from |
| valtree back to a MIR constant value must reintroduce an actual indirection. At codegen time the |
| addresses may be deduplicated between multiple uses or not, entirely depending on arbitrary |
| optimization choices.</p> |
| <p>As a consequence, all decoding of <code>ValTree</code> must happen by matching on the type first and making |
| decisions depending on that. The value itself gives no useful information without the type that |
| belongs to it.</p> |
| <p><a id="promoted"></a></p> |
| <h3 id="promoted-constants"><a class="header" href="#promoted-constants">Promoted constants</a></h3> |
| <p>See the const-eval WG's <a href="https://github.com/rust-lang/const-eval/blob/master/promotion.md">docs on promotion</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mir-construction"><a class="header" href="#mir-construction">MIR construction</a></h1> |
| <p>The lowering of <a href="mir/../hir.html">HIR</a> to <a href="mir/./index.html">MIR</a> occurs for the following (probably incomplete) |
| list of items:</p> |
| <ul> |
| <li>Function and closure bodies</li> |
| <li>Initializers of <code>static</code> and <code>const</code> items</li> |
| <li>Initializers of enum discriminants</li> |
| <li>Glue and shims of any kind |
| <ul> |
| <li>Tuple struct initializer functions</li> |
| <li>Drop code (the <code>Drop::drop</code> function is not called directly)</li> |
| <li>Drop implementations of types without an explicit <code>Drop</code> implementation</li> |
| </ul> |
| </li> |
| </ul> |
| <p>The lowering is triggered by calling the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/fn.mir_built.html"><code>mir_built</code></a> query. The MIR builder does |
| not actually use the HIR but operates on the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/index.html">THIR</a> instead, processing THIR |
| expressions recursively.</p> |
| <p>The lowering creates local variables for every argument as specified in the signature. |
| Next, it creates local variables for every binding specified (e.g. <code>(a, b): (i32, String)</code>) |
| produces 3 bindings, one for the argument, and two for the bindings. Next, it generates |
| field accesses that read the fields from the argument and writes the value to the binding |
| variable.</p> |
| <p>With this initialization out of the way, the lowering triggers a recursive call |
| to a function that generates the MIR for the body (a <code>Block</code> expression) and |
| writes the result into the <code>RETURN_PLACE</code>.</p> |
| <h2 id="unpack-all-the-things"><a class="header" href="#unpack-all-the-things"><code>unpack!</code> all the things</a></h2> |
| <p>Functions that generate MIR tend to fall into one of two patterns. |
| First, if the function generates only statements, then it will take a |
| basic block as argument onto which those statements should be appended. |
| It can then return a result as normal:</p> |
| <pre><code class="language-rust ignore">fn generate_some_mir(&mut self, block: BasicBlock) -> ResultType { |
| ... |
| }</code></pre> |
| <p>But there are other functions that may generate new basic blocks as well. |
| For example, lowering an expression like <code>if foo { 22 } else { 44 }</code> |
| requires generating a small "diamond-shaped graph". |
| In this case, the functions take a basic block where their code starts |
| and return a (potentially) new basic block where the code generation ends. |
| The <code>BlockAnd</code> type is used to represent this:</p> |
| <pre><code class="language-rust ignore">fn generate_more_mir(&mut self, block: BasicBlock) -> BlockAnd<ResultType> { |
| ... |
| }</code></pre> |
| <p>When you invoke these functions, it is common to have a local variable <code>block</code> |
| that is effectively a "cursor". It represents the point at which we are adding new MIR. |
| When you invoke <code>generate_more_mir</code>, you want to update this cursor. |
| You can do this manually, but it's tedious:</p> |
| <pre><code class="language-rust ignore">let mut block; |
| let v = match self.generate_more_mir(..) { |
| BlockAnd { block: new_block, value: v } => { |
| block = new_block; |
| v |
| } |
| };</code></pre> |
| <p>For this reason, we offer a macro that lets you write |
| <code>let v = unpack!(block = self.generate_more_mir(...))</code>. |
| It simply extracts the new block and overwrites the |
| variable <code>block</code> that you named in the <code>unpack!</code>.</p> |
| <h2 id="lowering-expressions-into-the-desired-mir"><a class="header" href="#lowering-expressions-into-the-desired-mir">Lowering expressions into the desired MIR</a></h2> |
| <p>There are essentially four kinds of representations one might want of an expression:</p> |
| <ul> |
| <li><code>Place</code> refers to a (or part of a) preexisting memory location (local, static, promoted)</li> |
| <li><code>Rvalue</code> is something that can be assigned to a <code>Place</code></li> |
| <li><code>Operand</code> is an argument to e.g. a <code>+</code> operation or a function call</li> |
| <li>a temporary variable containing a copy of the value</li> |
| </ul> |
| <p>The following image depicts a general overview of the interactions between the |
| representations:</p> |
| <img src="mir/mir_overview.svg"> |
| <p><a href="mir/mir_detailed.svg">Click here for a more detailed view</a></p> |
| <p>We start out with lowering the function body to an <code>Rvalue</code> so we can create an |
| assignment to <code>RETURN_PLACE</code>, This <code>Rvalue</code> lowering will in turn trigger lowering to |
| <code>Operand</code> for its arguments (if any). <code>Operand</code> lowering either produces a <code>const</code> |
| operand, or moves/copies out of a <code>Place</code>, thus triggering a <code>Place</code> lowering. An |
| expression being lowered to a <code>Place</code> can in turn trigger a temporary to be created |
| if the expression being lowered contains operations. This is where the snake bites its |
| own tail and we need to trigger an <code>Rvalue</code> lowering for the expression to be written |
| into the local.</p> |
| <h2 id="operator-lowering"><a class="header" href="#operator-lowering">Operator lowering</a></h2> |
| <p>Operators on builtin types are not lowered to function calls (which would end up being |
| infinite recursion calls, because the trait impls just contain the operation itself |
| again). Instead there are <code>Rvalue</code>s for binary and unary operators and index operations. |
| These <code>Rvalue</code>s later get codegened to llvm primitive operations or llvm intrinsics.</p> |
| <p>Operators on all other types get lowered to a function call to their <code>impl</code> of the |
| operator's corresponding trait.</p> |
| <p>Regardless of the lowering kind, the arguments to the operator are lowered to <code>Operand</code>s. |
| This means all arguments are either constants, or refer to an already existing value |
| somewhere in a local or static.</p> |
| <h2 id="method-call-lowering"><a class="header" href="#method-call-lowering">Method call lowering</a></h2> |
| <p>Method calls are lowered to the same <code>TerminatorKind</code> that function calls are. |
| In <a href="mir/./index.html">MIR</a> there is no difference between method calls and function calls anymore.</p> |
| <h2 id="conditions"><a class="header" href="#conditions">Conditions</a></h2> |
| <p><code>if</code> conditions and <code>match</code> statements for <code>enum</code>s with variants that have no fields are |
| lowered to <code>TerminatorKind::SwitchInt</code>. Each possible value (so <code>0</code> and <code>1</code> for <code>if</code> |
| conditions) has a corresponding <code>BasicBlock</code> to which the code continues. |
| The argument being branched on is (again) an <code>Operand</code> representing the value of |
| the if condition.</p> |
| <h3 id="pattern-matching"><a class="header" href="#pattern-matching">Pattern matching</a></h3> |
| <p><code>match</code> statements for <code>enum</code>s with variants that have fields are lowered to |
| <code>TerminatorKind::SwitchInt</code>, too, but the <code>Operand</code> refers to a <code>Place</code> where the |
| discriminant of the value can be found. This often involves reading the discriminant |
| to a new temporary variable.</p> |
| <h2 id="aggregate-construction"><a class="header" href="#aggregate-construction">Aggregate construction</a></h2> |
| <p>Aggregate values of any kind (e.g. structs or tuples) are built via <code>Rvalue::Aggregate</code>. |
| All fields are |
| lowered to <code>Operator</code>s. This is essentially equivalent to one assignment |
| statement per aggregate field plus an assignment to the discriminant in the |
| case of <code>enum</code>s.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mir-visitor"><a class="header" href="#mir-visitor">MIR visitor</a></h1> |
| <p>The MIR visitor is a convenient tool for traversing the MIR and either |
| looking for things or making changes to it. The visitor traits are |
| defined in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/visit/index.html">the <code>rustc_middle::mir::visit</code> module</a> – there are two of |
| them, generated via a single macro: <code>Visitor</code> (which operates on a |
| <code>&Mir</code> and gives back shared references) and <code>MutVisitor</code> (which |
| operates on a <code>&mut Mir</code> and gives back mutable references).</p> |
| <p>To implement a visitor, you have to create a type that represents |
| your visitor. Typically, this type wants to "hang on" to whatever |
| state you will need while processing MIR:</p> |
| <pre><code class="language-rust ignore">struct MyVisitor<...> { |
| tcx: TyCtxt<'tcx>, |
| ... |
| }</code></pre> |
| <p>and you then implement the <code>Visitor</code> or <code>MutVisitor</code> trait for that type:</p> |
| <pre><code class="language-rust ignore">impl<'tcx> MutVisitor<'tcx> for MyVisitor { |
| fn visit_foo(&mut self, ...) { |
| ... |
| self.super_foo(...); |
| } |
| }</code></pre> |
| <p>As shown above, within the impl, you can override any of the |
| <code>visit_foo</code> methods (e.g., <code>visit_terminator</code>) in order to write some |
| code that will execute whenever a <code>foo</code> is found. If you want to |
| recursively walk the contents of the <code>foo</code>, you then invoke the |
| <code>super_foo</code> method. (NB. You never want to override <code>super_foo</code>.)</p> |
| <p>A very simple example of a visitor can be found in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/prettify/struct.LocalFinder.html"><code>LocalFinder</code></a>. |
| By implementing <code>visit_local</code> method, this visitor identifies local variables that |
| can be candidates for reordering.</p> |
| <h2 id="traversal"><a class="header" href="#traversal">Traversal</a></h2> |
| <p>In addition the visitor, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/traversal/index.html">the <code>rustc_middle::mir::traversal</code> module</a> |
| contains useful functions for walking the MIR CFG in |
| <a href="https://en.wikipedia.org/wiki/Tree_traversal">different standard orders</a> (e.g. pre-order, reverse |
| post-order, and so forth).</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mir-queries-and-passes"><a class="header" href="#mir-queries-and-passes">MIR queries and passes</a></h1> |
| <p>If you would like to get the MIR:</p> |
| <ul> |
| <li>for a function - you can use the <code>optimized_mir</code> query (typically used by codegen) or the <code>mir_for_ctfe</code> query (typically used by compile time function evaluation, i.e., <em>CTFE</em>);</li> |
| <li>for a promoted - you can use the <code>promoted_mir</code> query.</li> |
| </ul> |
| <p>These will give you back the final, optimized MIR. For foreign def-ids, we simply read the MIR |
| from the other crate's metadata. But for local def-ids, the query will |
| construct the optimized MIR by requesting a pipeline of upstream queries<sup class="footnote-reference" id="fr-query-1"><a href="#footnote-query">1</a></sup>. |
| Each query will contain a series of passes. |
| This section describes how those queries and passes work and how you can extend them.</p> |
| <p>To produce the optimized MIR for a given def-id <code>D</code>, <code>optimized_mir(D)</code> |
| goes through several suites of passes, each grouped by a |
| query. Each suite consists of passes which perform linting, analysis, transformation or |
| optimization. Each query represent a useful intermediate point |
| where we can access the MIR dialect for type checking or other purposes:</p> |
| <ul> |
| <li><code>mir_built(D)</code> – it gives the initial MIR just after it's built;</li> |
| <li><code>mir_const(D)</code> – it applies some simple transformation passes to make MIR ready for |
| const qualification;</li> |
| <li><code>mir_promoted(D)</code> - it extracts promotable temps into separate MIR bodies, and also makes MIR |
| ready for borrow checking;</li> |
| <li><code>mir_drops_elaborated_and_const_checked(D)</code> - it performs borrow checking, runs major |
| transformation passes (such as drop elaboration) and makes MIR ready for optimization;</li> |
| <li><code>optimized_mir(D)</code> – it performs all enabled optimizations and reaches the final state.</li> |
| </ul> |
| <h2 id="implementing-and-registering-a-pass"><a class="header" href="#implementing-and-registering-a-pass">Implementing and registering a pass</a></h2> |
| <p>A <code>MirPass</code> is some bit of code that processes the MIR, typically transforming it along the way |
| somehow. But it may also do other things like linting (e.g., <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/check_packed_ref/struct.CheckPackedRef.html"><code>CheckPackedRef</code></a>, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/check_const_item_mutation/struct.CheckConstItemMutation.html"><code>CheckConstItemMutation</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/function_item_references/struct.FunctionItemReferences.html"><code>FunctionItemReferences</code></a>, which implement <code>MirLint</code>) or |
| optimization (e.g., <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/simplify/enum.SimplifyCfg.html"><code>SimplifyCfg</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/remove_unneeded_drops/struct.RemoveUnneededDrops.html"><code>RemoveUnneededDrops</code></a>). While most MIR passes |
| are defined in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/"><code>rustc_mir_transform</code></a> crate, the <code>MirPass</code> trait itself is |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/pass_manager/trait.MirPass.html">found</a> in the <code>rustc_middle</code> crate, and it basically consists of one primary method, |
| <code>run_pass</code>, that simply gets an <code>&mut Body</code> (along with the <code>tcx</code>). |
| The MIR is therefore modified in place (which helps to keep things efficient).</p> |
| <p>A basic example of a MIR pass is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/remove_storage_markers/struct.RemoveStorageMarkers.html"><code>RemoveStorageMarkers</code></a>, which walks |
| the MIR and removes all storage marks if they won't be emitted during codegen. As you |
| can see from its source, a MIR pass is defined by first defining a |
| dummy type, a struct with no fields:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>pub struct RemoveStorageMarkers; |
| <span class="boring">}</span></code></pre></pre> |
| <p>for which we implement the <code>MirPass</code> trait. We can then insert |
| this pass into the appropriate list of passes found in a query like |
| <code>mir_built</code>, <code>optimized_mir</code>, etc. (If this is an optimization, it |
| should go into the <code>optimized_mir</code> list.)</p> |
| <p>Another example of a simple MIR pass is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/cleanup_post_borrowck/struct.CleanupPostBorrowck.html"><code>CleanupPostBorrowck</code></a>, which walks |
| the MIR and removes all statements that are not relevant to code generation. As you can see from |
| its <a href="https://github.com/rust-lang/rust/blob/e2b52ff73edc8b0b7c74bc28760d618187731fe8/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs#L27">source</a>, it is defined by first defining a dummy type, a struct with no |
| fields:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>pub struct CleanupPostBorrowck; |
| <span class="boring">}</span></code></pre></pre> |
| <p>for which we implement the <code>MirPass</code> trait:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>impl<'tcx> MirPass<'tcx> for CleanupPostBorrowck { |
| fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
| ... |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>We <a href="https://github.com/rust-lang/rust/blob/e2b52ff73edc8b0b7c74bc28760d618187731fe8/compiler/rustc_mir_transform/src/lib.rs#L413">register</a> this pass inside the <code>mir_drops_elaborated_and_const_checked</code> query. |
| (If this is an optimization, it should go into the <code>optimized_mir</code> list.)</p> |
| <p>If you are writing a pass, there's a good chance that you are going to |
| want to use a <a href="mir/./visitor.html">MIR visitor</a>. MIR visitors are a handy way to walk all |
| the parts of the MIR, either to search for something or to make small |
| edits.</p> |
| <h2 id="stealing"><a class="header" href="#stealing">Stealing</a></h2> |
| <p>The intermediate queries <code>mir_const()</code> and <code>mir_promoted()</code> yield up |
| a <code>&'tcx Steal<Body<'tcx>></code>, allocated using <code>tcx.alloc_steal_mir()</code>. |
| This indicates that the result may be <strong>stolen</strong> by a subsequent query – this is an |
| optimization to avoid cloning the MIR. Attempting to use a stolen |
| result will cause a panic in the compiler. Therefore, it is important |
| that you do not accidentally read from these intermediate queries without |
| the consideration of the dependency in the MIR processing pipeline.</p> |
| <p>Because of this stealing mechanism, some care must be taken to |
| ensure that, before the MIR at a particular phase in the processing |
| pipeline is stolen, anyone who may want to read from it has already |
| done so.</p> |
| <p>Concretely, this means that if you have a query <code>foo(D)</code> |
| that wants to access the result of <code>mir_promoted(D)</code>, you need to have <code>foo(D)</code> |
| calling the <code>mir_const(D)</code> query first. This will force it |
| to execute even though you don't directly require its result.</p> |
| <blockquote> |
| <p>This mechanism is a bit dodgy. There is a discussion of more elegant |
| alternatives in <a href="https://github.com/rust-lang/rust/issues/41710">rust-lang/rust#41710</a>.</p> |
| </blockquote> |
| <h3 id="overview-3"><a class="header" href="#overview-3">Overview</a></h3> |
| <p>Below is an overview of the stealing dependency in the MIR processing pipeline<sup class="footnote-reference" id="fr-part-1"><a href="#footnote-part">2</a></sup>:</p> |
| <pre class="mermaid">flowchart BT |
| mir_for_ctfe* --borrow--> id40 |
| id5 --steal--> id40 |
| |
| mir_borrowck* --borrow--> id3 |
| id41 --steal part 1--> id3 |
| id40 --steal part 0--> id3 |
| |
| mir_const_qualif* -- borrow --> id2 |
| id3 -- steal --> id2 |
| |
| id2 -- steal --> id1 |
| |
| id1([mir_built]) |
| id2([mir_const]) |
| id3([mir_promoted]) |
| id40([mir_drops_elaborated_and_const_checked]) |
| id41([promoted_mir]) |
| id5([optimized_mir]) |
| |
| style id1 fill:#bbf |
| style id2 fill:#bbf |
| style id3 fill:#bbf |
| style id40 fill:#bbf |
| style id41 fill:#bbf |
| style id5 fill:#bbf |
| </pre> |
| <p>The stadium-shape queries (e.g., <code>mir_built</code>) with a deep color are the primary queries in the |
| pipeline, while the rectangle-shape queries (e.g., <code>mir_const_qualif*</code><sup class="footnote-reference" id="fr-star-1"><a href="#footnote-star">3</a></sup>) with a shallow color |
| are those subsequent queries that need to read the results from <code>&'tcx Steal<Body<'tcx>></code>. With the |
| stealing mechanism, the rectangle-shape queries must be performed before any stadium-shape queries, |
| that have an equal or larger height in the dependency tree, ever do.</p> |
| <h3 id="example-1"><a class="header" href="#example-1">Example</a></h3> |
| <p>As an example, consider MIR const qualification. It wants to read the result produced by the |
| <code>mir_const</code> query. However, that result will be <strong>stolen</strong> by the <code>mir_promoted</code> query at some |
| time in the pipeline. Before <code>mir_promoted</code> is ever queried, calling the <code>mir_const_qualif</code> query |
| will succeed since <code>mir_const</code> will produce (if queried the first time) or cache (if queried |
| multiple times) the <code>Steal</code> result and the result is <strong>not</strong> stolen yet. After <code>mir_promoted</code> is |
| queried, the result would be stolen and calling the <code>mir_const_qualif</code> query to read the result |
| would cause a panic.</p> |
| <p>Therefore, with this stealing mechanism, <code>mir_promoted</code> should guarantee any <code>mir_const_qualif*</code> |
| queries are called before it actually steals, thus ensuring that the reads have already happened |
| (remember that <a href="mir/../query.html">queries are memoized</a>, so executing a query twice |
| simply loads from a cache the second time).</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-query"> |
| <p>See the <a href="mir/../query.html">Queries</a> chapter for the general concept of query. <a href="#fr-query-1">↩</a></p> |
| </li> |
| <li id="footnote-part"> |
| <p>The <code>mir_promoted</code> query will yield up a tuple |
| <code>(&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>)</code>, <code>promoted_mir</code> will steal |
| part 1 (<code>&'tcx Steal<IndexVec<Promoted, Body<'tcx>>></code>) and <code>mir_drops_elaborated_and_const_checked</code> |
| will steal part 0 (<code>&'tcx Steal<Body<'tcx>></code>). And their stealing is irrelevant to each other, |
| i.e., can be performed separately. <a href="#fr-part-1">↩</a></p> |
| </li> |
| <li id="footnote-star"> |
| <p>Note that the <code>*</code> suffix in the queries represent a set of queries with the same prefix. |
| For example, <code>mir_borrowck*</code> represents <code>mir_borrowck</code>, <code>mir_borrowck_const_arg</code> and |
| <code>mir_borrowck_opt_const_arg</code>. <a href="#fr-star-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="inline-assembly"><a class="header" href="#inline-assembly">Inline assembly</a></h1> |
| <h2 id="overview-4"><a class="header" href="#overview-4">Overview</a></h2> |
| <p>Inline assembly in rustc mostly revolves around taking an <code>asm!</code> macro invocation and plumbing it |
| through all of the compiler layers down to LLVM codegen. Throughout the various stages, an |
| <code>InlineAsm</code> generally consists of 3 components:</p> |
| <ul> |
| <li> |
| <p>The template string, which is stored as an array of <code>InlineAsmTemplatePiece</code>. Each piece |
| represents either a literal or a placeholder for an operand (just like format strings).</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>pub enum InlineAsmTemplatePiece { |
| String(String), |
| Placeholder { operand_idx: usize, modifier: Option<char>, span: Span }, |
| } |
| <span class="boring">}</span></code></pre></pre> |
| </li> |
| <li> |
| <p>The list of operands to the <code>asm!</code> (<code>in</code>, <code>[late]out</code>, <code>in[late]out</code>, <code>sym</code>, <code>const</code>). These are |
| represented differently at each stage of lowering, but follow a common pattern:</p> |
| <ul> |
| <li><code>in</code>, <code>out</code> and <code>inout</code> all have an associated register class (<code>reg</code>) or explicit register |
| (<code>"eax"</code>).</li> |
| <li><code>inout</code> has 2 forms: one with a single expression that is both read from and written to, and |
| one with two separate expressions for the input and output parts.</li> |
| <li><code>out</code> and <code>inout</code> have a <code>late</code> flag (<code>lateout</code> / <code>inlateout</code>) to indicate that the register |
| allocator is allowed to reuse an input register for this output.</li> |
| <li><code>out</code> and the split variant of <code>inout</code> allow <code>_</code> to be specified for an output, which means |
| that the output is discarded. This is used to allocate scratch registers for assembly code.</li> |
| <li><code>const</code> refers to an anonymous constants and generally works like an inline const.</li> |
| <li><code>sym</code> is a bit special since it only accepts a path expression, which must point to a <code>static</code> |
| or a <code>fn</code>.</li> |
| </ul> |
| </li> |
| <li> |
| <p>The options set at the end of the <code>asm!</code> macro. The only ones that are of particular interest to |
| rustc are <code>NORETURN</code> which makes <code>asm!</code> return <code>!</code> instead of <code>()</code>, and <code>RAW</code> which disables format |
| string parsing. The remaining options are mostly passed through to LLVM with little processing.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>bitflags::bitflags! { |
| pub struct InlineAsmOptions: u16 { |
| const PURE = 1 << 0; |
| const NOMEM = 1 << 1; |
| const READONLY = 1 << 2; |
| const PRESERVES_FLAGS = 1 << 3; |
| const NORETURN = 1 << 4; |
| const NOSTACK = 1 << 5; |
| const ATT_SYNTAX = 1 << 6; |
| const RAW = 1 << 7; |
| const MAY_UNWIND = 1 << 8; |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| </li> |
| </ul> |
| <h2 id="ast"><a class="header" href="#ast">AST</a></h2> |
| <p><code>InlineAsm</code> is represented as an expression in the AST with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ast/struct.InlineAsm.html"><code>ast::InlineAsm</code> type</a>.</p> |
| <p>The <code>asm!</code> macro is implemented in <code>rustc_builtin_macros</code> and outputs an <code>InlineAsm</code> AST node. The |
| template string is parsed using <code>fmt_macros</code>, positional and named operands are resolved to |
| explicit operand indices. Since target information is not available to macro invocations, |
| validation of the registers and register classes is deferred to AST lowering.</p> |
| <h2 id="hir"><a class="header" href="#hir">HIR</a></h2> |
| <p><code>InlineAsm</code> is represented as an expression in the HIR with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.InlineAsm.html"><code>hir::InlineAsm</code> type</a>.</p> |
| <p>AST lowering is where <code>InlineAsmRegOrRegClass</code> is converted from <code>Symbol</code>s to an actual register or |
| register class. If any modifiers are specified for a template string placeholder, these are |
| validated against the set allowed for that operand type. Finally, explicit registers for inputs and |
| outputs are checked for conflicts (same register used for different operands).</p> |
| <h2 id="type-checking"><a class="header" href="#type-checking">Type checking</a></h2> |
| <p>Each register class has a whitelist of types that it may be used with. After the types of all |
| operands have been determined, the <code>intrinsicck</code> pass will check that these types are in the |
| whitelist. It also checks that split <code>inout</code> operands have compatible types and that <code>const</code> |
| operands are integers or floats. Suggestions are emitted where needed if a template modifier should |
| be used for an operand based on the type that was passed into it.</p> |
| <h2 id="thir"><a class="header" href="#thir">THIR</a></h2> |
| <p><code>InlineAsm</code> is represented as an expression in the THIR with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/thir/struct.InlineAsmExpr.html"><code>InlineAsmExpr</code> type</a>.</p> |
| <p>The only significant change compared to HIR is that <code>Sym</code> has been lowered to either a <code>SymFn</code> |
| whose <code>expr</code> is a <code>Literal</code> ZST of the <code>fn</code>, or a <code>SymStatic</code> which points to the <code>DefId</code> of a |
| <code>static</code>.</p> |
| <h2 id="mir"><a class="header" href="#mir">MIR</a></h2> |
| <p><code>InlineAsm</code> is represented as a <code>Terminator</code> in the MIR with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.TerminatorKind.html#variant.InlineAsm"><code>TerminatorKind::InlineAsm</code> variant</a></p> |
| <p>As part of THIR lowering, <code>InOut</code> and <code>SplitInOut</code> operands are lowered to a split form with a |
| separate <code>in_value</code> and <code>out_place</code>.</p> |
| <p>Semantically, the <code>InlineAsm</code> terminator is similar to the <code>Call</code> terminator except that it has |
| multiple output places where a <code>Call</code> only has a single return place output.</p> |
| <h2 id="codegen"><a class="header" href="#codegen">Codegen</a></h2> |
| <p>Operands are lowered one more time before being passed to LLVM codegen, this is represented by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/traits/enum.InlineAsmOperandRef.html"><code>InlineAsmOperandRef</code> type</a> from <code>rustc_codegen_ssa</code>.</p> |
| <p>The operands are lowered to LLVM operands and constraint codes as follows:</p> |
| <ul> |
| <li><code>out</code> and the output part of <code>inout</code> operands are added first, as required by LLVM. Late output |
| operands have a <code>=</code> prefix added to their constraint code, non-late output operands have a <code>=&</code> |
| prefix added to their constraint code.</li> |
| <li><code>in</code> operands are added normally.</li> |
| <li><code>inout</code> operands are tied to the matching output operand.</li> |
| <li><code>sym</code> operands are passed as function pointers or pointers, using the <code>"s"</code> constraint.</li> |
| <li><code>const</code> operands are formatted to a string and directly inserted in the template string.</li> |
| </ul> |
| <p>The template string is converted to LLVM form:</p> |
| <ul> |
| <li><code>$</code> characters are escaped as <code>$$</code>.</li> |
| <li><code>const</code> operands are converted to strings and inserted directly.</li> |
| <li>Placeholders are formatted as <code>${X:M}</code> where <code>X</code> is the operand index and <code>M</code> is the modifier |
| character. Modifiers are converted from the Rust form to the LLVM form.</li> |
| </ul> |
| <p>The various options are converted to clobber constraints or LLVM attributes, refer to the |
| <a href="https://github.com/Amanieu/rfcs/blob/inline-asm/text/0000-inline-asm.md#mapping-to-llvm-ir">RFC</a> |
| for more details.</p> |
| <p>Note that LLVM is sometimes rather picky about what types it accepts for certain constraint codes |
| so we sometimes need to insert conversions to/from a supported type. See the target-specific |
| ISelLowering.cpp files in LLVM for details of what types are supported for each register class.</p> |
| <h2 id="adding-support-for-new-architectures"><a class="header" href="#adding-support-for-new-architectures">Adding support for new architectures</a></h2> |
| <p>Adding inline assembly support to an architecture is mostly a matter of defining the registers and |
| register classes for that architecture. All the definitions for register classes are located in |
| <code>compiler/rustc_target/asm/</code>.</p> |
| <p>Additionally you will need to implement lowering of these register classes to LLVM constraint codes |
| in <code>compiler/rustc_codegen_llvm/asm.rs</code>.</p> |
| <p>When adding a new architecture, make sure to cross-reference with the LLVM source code:</p> |
| <ul> |
| <li>LLVM has restrictions on which types can be used with a particular constraint code. Refer to the |
| <code>getRegForInlineAsmConstraint</code> function in <code>lib/Target/${ARCH}/${ARCH}ISelLowering.cpp</code>.</li> |
| <li>LLVM reserves certain registers for its internal use, which causes them to not be saved/restored |
| properly around inline assembly blocks. These registers are listed in the <code>getReservedRegs</code> |
| function in <code>lib/Target/${ARCH}/${ARCH}RegisterInfo.cpp</code>. Any "conditionally" reserved register |
| such as the frame/base pointer must always be treated as reserved for Rust purposes because we |
| can't know ahead of time whether a function will require a frame/base pointer.</li> |
| </ul> |
| <h2 id="tests-2"><a class="header" href="#tests-2">Tests</a></h2> |
| <p>Various tests for inline assembly are available:</p> |
| <ul> |
| <li><code>tests/assembly-llvm/asm</code></li> |
| <li><code>tests/ui/asm</code></li> |
| <li><code>tests/codegen-llvm/asm-*</code></li> |
| </ul> |
| <p>Every architecture supported by inline assembly must have exhaustive tests in |
| <code>tests/assembly-llvm/asm</code> which test all combinations of register classes and types.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="command-line-arguments"><a class="header" href="#command-line-arguments">Command-line Arguments</a></h1> |
| <p>Command-line flags are documented in the <a href="https://doc.rust-lang.org/rustc/command-line-arguments.html">rustc book</a>. All <em>stable</em> |
| flags should be documented there. Unstable flags should be documented in the |
| <a href="https://doc.rust-lang.org/nightly/unstable-book/">unstable book</a>.</p> |
| <p>See the <a href="https://forge.rust-lang.org/compiler/proposals-and-stabilization.html#compiler-flags">forge guide for new options</a> for details on the <em>procedure</em> for |
| adding a new command-line argument.</p> |
| <h2 id="guidelines-1"><a class="header" href="#guidelines-1">Guidelines</a></h2> |
| <ul> |
| <li>Flags should be orthogonal to each other. For example, if we'd have a |
| json-emitting variant of multiple actions <code>foo</code> and <code>bar</code>, an additional |
| <code>--json</code> flag is better than adding <code>--foo-json</code> and <code>--bar-json</code>.</li> |
| <li>Avoid flags with the <code>no-</code> prefix. Instead, use the <a href="https://github.com/rust-lang/rust/blob/e5335592e78354e33d798d20c04bcd677c1df62d/src/librustc_session/options.rs#L307-L313"><code>parse_bool</code></a> function, |
| such as <code>-C embed-bitcode=no</code>.</li> |
| <li>Consider the behavior if the flag is passed multiple times. In some |
| situations, the values should be accumulated (in order!). In other |
| situations, subsequent flags should override previous flags (for example, |
| the lint-level flags). And some flags (like <code>-o</code>) should generate an error |
| if it is too ambiguous what multiple flags would mean.</li> |
| <li>Always give options a long descriptive name, if only for more understandable |
| compiler scripts.</li> |
| <li>The <code>--verbose</code> flag is for adding verbose information to <code>rustc</code> |
| output. For example, using it with the <code>--version</code> |
| flag gives information about the hashes of the compiler code.</li> |
| <li>Experimental flags and options must be guarded behind the <code>-Z unstable-options</code> flag.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rustc_driver-and-rustc_interface"><a class="header" href="#rustc_driver-and-rustc_interface"><code>rustc_driver</code> and <code>rustc_interface</code></a></h1> |
| <h2 id="rustc_driver"><a class="header" href="#rustc_driver"><code>rustc_driver</code></a></h2> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/"><code>rustc_driver</code></a> is essentially <code>rustc</code>'s <code>main</code> function. |
| It acts as the glue for running the various phases of the compiler in the correct order, |
| using the interface defined in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> crate. Where possible, using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/"><code>rustc_driver</code></a> rather than <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> is recommended.</p> |
| <p>The main entry point of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/"><code>rustc_driver</code></a> is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/fn.run_compiler.html"><code>rustc_driver::run_compiler</code></a>. |
| This builder accepts the same command-line args as rustc as well as an implementation of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/trait.Callbacks.html"><code>Callbacks</code></a> and a couple of other optional options. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/trait.Callbacks.html"><code>Callbacks</code></a> is a <code>trait</code> that allows for custom compiler configuration, |
| as well as allowing custom code to run after different phases of the compilation.</p> |
| <h2 id="rustc_interface"><a class="header" href="#rustc_interface"><code>rustc_interface</code></a></h2> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> crate provides a low level API to external users for manually driving the compilation process, |
| allowing third parties to effectively use <code>rustc</code>'s internals as a library for analyzing a crate or for ad hoc emulating of the compiler for cases where <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/"><code>rustc_driver</code></a> is not flexible enough (i.e. <code>rustdoc</code> compiling code and serving output).</p> |
| <p>The main entry point of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/fn.run_compiler.html"><code>rustc_interface::run_compiler</code></a>) takes a configuration variable for the compiler |
| and a <code>closure</code> taking a yet unresolved <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Compiler.html"><code>Compiler</code></a>. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/fn.run_compiler.html"><code>run_compiler</code></a> creates a <code>Compiler</code> from the configuration and passes it to the <code>closure</code>. |
| Inside the <code>closure</code> you can use the <code>Compiler</code> to call various functions to compile a crate and get the results. |
| You can see a minimal example of how to use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> <a href="https://github.com/rust-lang/rustc-dev-guide/blob/main/examples/rustc-interface-example.rs">here</a>.</p> |
| <p>You can see an example of how to use the various functions using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> needs by looking at the <code>rustc_driver</code> implementation, |
| specifically <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver_impl/fn.run_compiler.html"><code>rustc_driver_impl::run_compiler</code></a> |
| (not to be confused with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/fn.run_compiler.html"><code>rustc_interface::run_compiler</code></a>).</p> |
| <blockquote> |
| <p><strong>Warning:</strong> By its very nature, the internal compiler APIs are always going |
| to be unstable. That said, we do try not to break things unnecessarily.</p> |
| </blockquote> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="remarks-on-perma-unstable-features"><a class="header" href="#remarks-on-perma-unstable-features">Remarks on perma unstable features</a></h1> |
| <h2 id="rustc_private"><a class="header" href="#rustc_private"><code>rustc_private</code></a></h2> |
| <h3 id="overview-5"><a class="header" href="#overview-5">Overview</a></h3> |
| <p>The <code>rustc_private</code> feature allows external crates to use compiler internals.</p> |
| <h3 id="using-rustc-private-with-official-toolchains"><a class="header" href="#using-rustc-private-with-official-toolchains">Using <code>rustc-private</code> with Official Toolchains</a></h3> |
| <p>When using the <code>rustc_private</code> feature with official Rust toolchains distributed via rustup, you need to install two additional components:</p> |
| <ol> |
| <li><strong><code>rustc-dev</code></strong>: Provides compiler libraries</li> |
| <li><strong><code>llvm-tools</code></strong>: Provides LLVM libraries required for linking</li> |
| </ol> |
| <h4 id="installation-steps"><a class="header" href="#installation-steps">Installation Steps</a></h4> |
| <p>Install both components using rustup:</p> |
| <pre><code class="language-text">rustup component add rustc-dev llvm-tools |
| </code></pre> |
| <h4 id="common-error"><a class="header" href="#common-error">Common Error</a></h4> |
| <p>Without the <code>llvm-tools</code> component, you'll encounter linking errors like:</p> |
| <pre><code class="language-text">error: linking with `cc` failed: exit status: 1 |
| | |
| = note: rust-lld: error: unable to find library -lLLVM-{version} |
| </code></pre> |
| <h3 id="using-rustc-private-with-custom-toolchains"><a class="header" href="#using-rustc-private-with-custom-toolchains">Using <code>rustc-private</code> with Custom Toolchains</a></h3> |
| <p>For custom-built toolchains or environments not using rustup, additional configuration is typically required:</p> |
| <h4 id="requirements"><a class="header" href="#requirements">Requirements</a></h4> |
| <ul> |
| <li>LLVM libraries must be available in your system's library search paths</li> |
| <li>The LLVM version must match the one used to build your Rust toolchain</li> |
| </ul> |
| <h4 id="troubleshooting-steps"><a class="header" href="#troubleshooting-steps">Troubleshooting Steps</a></h4> |
| <ol> |
| <li><strong>Check LLVM installation</strong>: Verify LLVM is installed and accessible</li> |
| <li><strong>Configure library paths</strong>: You may need to set environment variables: |
| <pre><code class="language-text">export LD_LIBRARY_PATH=/path/to/llvm/lib:$LD_LIBRARY_PATH |
| </code></pre> |
| </li> |
| <li><strong>Check version compatibility</strong>: Ensure your LLVM version is compatible with your Rust toolchain</li> |
| </ol> |
| <h3 id="additional-resources"><a class="header" href="#additional-resources">Additional Resources</a></h3> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/issues/137421">GitHub Issue #137421</a>: Explains that <code>rustc_private</code> linker failures often occur because <code>llvm-tools</code> is not installed</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="example-type-checking-through-rustc_driver"><a class="header" href="#example-type-checking-through-rustc_driver">Example: Type checking through <code>rustc_driver</code></a></h1> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver"><code>rustc_driver</code></a> allows you to interact with Rust code at various stages of compilation.</p> |
| <h2 id="getting-the-type-of-an-expression"><a class="header" href="#getting-the-type-of-an-expression">Getting the type of an expression</a></h2> |
| <p>To get the type of an expression, use the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/trait.Callbacks.html#method.after_analysis"><code>after_analysis</code></a> callback to get a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html"><code>TyCtxt</code></a>.</p> |
| <pre><pre class="playground"><code class="language-rust">// Tested with nightly-2025-03-28 |
| |
| #![feature(rustc_private)] |
| |
| extern crate rustc_ast; |
| extern crate rustc_ast_pretty; |
| extern crate rustc_data_structures; |
| extern crate rustc_driver; |
| extern crate rustc_error_codes; |
| extern crate rustc_errors; |
| extern crate rustc_hash; |
| extern crate rustc_hir; |
| extern crate rustc_interface; |
| extern crate rustc_middle; |
| extern crate rustc_session; |
| extern crate rustc_span; |
| |
| use std::io; |
| use std::path::Path; |
| use std::sync::Arc; |
| |
| use rustc_ast_pretty::pprust::item_to_string; |
| use rustc_driver::{Compilation, run_compiler}; |
| use rustc_interface::interface::{Compiler, Config}; |
| use rustc_middle::ty::TyCtxt; |
| |
| struct MyFileLoader; |
| |
| impl rustc_span::source_map::FileLoader for MyFileLoader { |
| fn file_exists(&self, path: &Path) -> bool { |
| path == Path::new("main.rs") |
| } |
| |
| fn read_file(&self, path: &Path) -> io::Result<String> { |
| if path == Path::new("main.rs") { |
| Ok(r#" |
| fn main() { |
| let message = "Hello, World!"; |
| println!("{message}"); |
| } |
| "# |
| .to_string()) |
| } else { |
| Err(io::Error::other("oops")) |
| } |
| } |
| |
| fn read_binary_file(&self, _path: &Path) -> io::Result<Arc<[u8]>> { |
| Err(io::Error::other("oops")) |
| } |
| } |
| |
| struct MyCallbacks; |
| |
| impl rustc_driver::Callbacks for MyCallbacks { |
| fn config(&mut self, config: &mut Config) { |
| config.file_loader = Some(Box::new(MyFileLoader)); |
| } |
| |
| fn after_crate_root_parsing( |
| &mut self, |
| _compiler: &Compiler, |
| krate: &mut rustc_ast::Crate, |
| ) -> Compilation { |
| for item in &krate.items { |
| println!("{}", item_to_string(&item)); |
| } |
| |
| Compilation::Continue |
| } |
| |
| fn after_analysis(&mut self, _compiler: &Compiler, tcx: TyCtxt<'_>) -> Compilation { |
| // Iterate over the top-level items in the crate, looking for the main function. |
| for id in tcx.hir_free_items() { |
| let item = &tcx.hir_item(id); |
| // Use pattern-matching to find a specific node inside the main function. |
| if let rustc_hir::ItemKind::Fn { body, .. } = item.kind { |
| let expr = &tcx.hir_body(body).value; |
| if let rustc_hir::ExprKind::Block(block, _) = expr.kind { |
| if let rustc_hir::StmtKind::Let(let_stmt) = block.stmts[0].kind { |
| if let Some(expr) = let_stmt.init { |
| let hir_id = expr.hir_id; // hir_id identifies the string "Hello, world!" |
| let def_id = item.hir_id().owner.def_id; // def_id identifies the main function |
| let ty = tcx.typeck(def_id).node_type(hir_id); |
| println!("{expr:#?}: {ty:?}"); |
| } |
| } |
| } |
| } |
| } |
| |
| Compilation::Stop |
| } |
| } |
| |
| fn main() { |
| run_compiler( |
| &[ |
| // The first argument, which in practice contains the name of the binary being executed |
| // (i.e. "rustc") is ignored by rustc. |
| "ignored".to_string(), |
| "main.rs".to_string(), |
| ], |
| &mut MyCallbacks, |
| ); |
| }</code></pre></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="example-getting-diagnostic-through-rustc_interface"><a class="header" href="#example-getting-diagnostic-through-rustc_interface">Example: Getting diagnostic through <code>rustc_interface</code></a></h1> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html"><code>rustc_interface</code></a> allows you to intercept diagnostics that would |
| otherwise be printed to stderr.</p> |
| <h2 id="getting-diagnostics"><a class="header" href="#getting-diagnostics">Getting diagnostics</a></h2> |
| <p>To get diagnostics from the compiler, |
| configure <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Config.html"><code>rustc_interface::Config</code></a> to output diagnostic to a buffer, |
| and run <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn.typeck.html"><code>rustc_hir_typeck::typeck</code></a> for each item.</p> |
| <pre><pre class="playground"><code class="language-rust">// Tested with nightly-2025-03-28 |
| |
| #![feature(rustc_private)] |
| |
| extern crate rustc_data_structures; |
| extern crate rustc_driver; |
| extern crate rustc_error_codes; |
| extern crate rustc_errors; |
| extern crate rustc_hash; |
| extern crate rustc_hir; |
| extern crate rustc_interface; |
| extern crate rustc_session; |
| extern crate rustc_span; |
| |
| use std::sync::{Arc, Mutex}; |
| |
| use rustc_errors::emitter::Emitter; |
| use rustc_errors::registry::{self, Registry}; |
| use rustc_errors::translation::Translate; |
| use rustc_errors::{DiagInner, FluentBundle}; |
| use rustc_session::config; |
| use rustc_span::source_map::SourceMap; |
| |
| struct DebugEmitter { |
| source_map: Arc<SourceMap>, |
| diagnostics: Arc<Mutex<Vec<DiagInner>>>, |
| } |
| |
| impl Translate for DebugEmitter { |
| fn fluent_bundle(&self) -> Option<&FluentBundle> { |
| None |
| } |
| |
| fn fallback_fluent_bundle(&self) -> &FluentBundle { |
| panic!("this emitter should not translate message") |
| } |
| } |
| |
| impl Emitter for DebugEmitter { |
| fn emit_diagnostic(&mut self, diag: DiagInner, _: &Registry) { |
| self.diagnostics.lock().unwrap().push(diag); |
| } |
| |
| fn source_map(&self) -> Option<&SourceMap> { |
| Some(&self.source_map) |
| } |
| } |
| |
| fn main() { |
| let buffer: Arc<Mutex<Vec<DiagInner>>> = Arc::default(); |
| let diagnostics = buffer.clone(); |
| let config = rustc_interface::Config { |
| opts: config::Options::default(), |
| // This program contains a type error. |
| input: config::Input::Str { |
| name: rustc_span::FileName::Custom("main.rs".into()), |
| input: " |
| fn main() { |
| let x: &str = 1; |
| } |
| " |
| .into(), |
| }, |
| crate_cfg: Vec::new(), |
| crate_check_cfg: Vec::new(), |
| output_dir: None, |
| output_file: None, |
| file_loader: None, |
| locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_owned(), |
| lint_caps: rustc_hash::FxHashMap::default(), |
| psess_created: Some(Box::new(|parse_sess| { |
| parse_sess.dcx().set_emitter(Box::new(DebugEmitter { |
| source_map: parse_sess.clone_source_map(), |
| diagnostics, |
| })); |
| })), |
| register_lints: None, |
| override_queries: None, |
| registry: registry::Registry::new(rustc_errors::codes::DIAGNOSTICS), |
| make_codegen_backend: None, |
| expanded_args: Vec::new(), |
| ice_file: None, |
| hash_untracked_state: None, |
| using_internal_features: &rustc_driver::USING_INTERNAL_FEATURES, |
| }; |
| rustc_interface::run_compiler(config, |compiler| { |
| let krate = rustc_interface::passes::parse(&compiler.sess); |
| rustc_interface::create_and_enter_global_ctxt(&compiler, krate, |tcx| { |
| // Iterate all the items defined and perform type checking. |
| tcx.par_hir_body_owners(|item_def_id| { |
| tcx.ensure_ok().typeck(item_def_id); |
| }); |
| }); |
| // If the compiler has encountered errors when this closure returns, it will abort (!) the program. |
| // We avoid this by resetting the error count before returning |
| compiler.sess.dcx().reset_err_count(); |
| }); |
| // Read buffered diagnostics. |
| buffer.lock().unwrap().iter().for_each(|diagnostic| { |
| println!("{diagnostic:#?}"); |
| }); |
| }</code></pre></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="errors-and-lints"><a class="header" href="#errors-and-lints">Errors and lints</a></h1> |
| <p>A lot of effort has been put into making <code>rustc</code> have great error messages. |
| This chapter is about how to emit compile errors and lints from the compiler.</p> |
| <h2 id="diagnostic-structure"><a class="header" href="#diagnostic-structure">Diagnostic structure</a></h2> |
| <p>The main parts of a diagnostic error are the following:</p> |
| <pre><code>error[E0000]: main error message |
| --> file.rs:LL:CC |
| | |
| LL | <code> |
| | -^^^^- secondary label |
| | | |
| | primary label |
| | |
| = note: note without a `Span`, created with `.note` |
| note: sub-diagnostic message for `.span_note` |
| --> file.rs:LL:CC |
| | |
| LL | more code |
| | ^^^^ |
| </code></pre> |
| <ul> |
| <li>Level (<code>error</code>, <code>warning</code>, etc.). It indicates the severity of the message. |
| (See <a href="diagnostics.html#diagnostic-levels">diagnostic levels</a>)</li> |
| <li>Code (for example, for "mismatched types", it is <code>E0308</code>). It helps |
| users get more information about the current error through an extended |
| description of the problem in the error code index. Not all diagnostic have a |
| code. For example, diagnostics created by lints don't have one.</li> |
| <li>Message. It is the main description of the problem. It should be general and |
| able to stand on its own, so that it can make sense even in isolation.</li> |
| <li>Diagnostic window. This contains several things: |
| <ul> |
| <li>The path, line number and column of the beginning of the primary span.</li> |
| <li>The users' affected code and its surroundings.</li> |
| <li>Primary and secondary spans underlying the users' code. These spans can |
| optionally contain one or more labels. |
| <ul> |
| <li>Primary spans should have enough text to describe the problem in such a |
| way that if it were the only thing being displayed (for example, in an |
| IDE) it would still make sense. Because it is "spatially aware" (it |
| points at the code), it can generally be more succinct than the error |
| message.</li> |
| <li>If cluttered output can be foreseen in cases when multiple span labels |
| overlap, it is a good idea to tweak the output appropriately. For |
| example, the <code>if/else arms have incompatible types</code> error uses different |
| spans depending on whether the arms are all in the same line, if one of |
| the arms is empty and if none of those cases applies.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li>Sub-diagnostics. Any error can have multiple sub-diagnostics that look |
| similar to the main part of the error. These are used for cases where the |
| order of the explanation might not correspond with the order of the code. If |
| the order of the explanation can be "order free", leveraging secondary labels |
| in the main diagnostic is preferred, as it is typically less verbose.</li> |
| </ul> |
| <p>The text should be matter of fact and avoid capitalization and periods, unless |
| multiple sentences are <em>needed</em>:</p> |
| <pre><code class="language-txt">error: the fobrulator needs to be krontrificated |
| </code></pre> |
| <p>When code or an identifier must appear in a message or label, it should be |
| surrounded with backticks:</p> |
| <pre><code class="language-txt">error: the identifier `foo.bar` is invalid |
| </code></pre> |
| <h3 id="error-codes-and-explanations"><a class="header" href="#error-codes-and-explanations">Error codes and explanations</a></h3> |
| <p>Most errors have an associated error code. Error codes are linked to long-form |
| explanations which contains an example of how to trigger the error and in-depth |
| details about the error. They may be viewed with the <code>--explain</code> flag, or via |
| the <a href="https://doc.rust-lang.org/error-index.html">error index</a>.</p> |
| <p>As a general rule, give an error a code (with an associated explanation) if the |
| explanation would give more information than the error itself. A lot of the time |
| it's better to put all the information in the emitted error itself. However, |
| sometimes that would make the error verbose or there are too many possible |
| triggers to include useful information for all cases in the error, in which case |
| it's a good idea to add an explanation.<sup class="footnote-reference" id="fr-estebank-1"><a href="#footnote-estebank">1</a></sup> |
| As always, if you are not sure, just ask your reviewer!</p> |
| <p>If you decide to add a new error with an associated error code, please read |
| <a href="./diagnostics/error-codes.html">this section</a> for a guide and important details about the |
| process.</p> |
| <h3 id="lints-versus-fixed-diagnostics"><a class="header" href="#lints-versus-fixed-diagnostics">Lints versus fixed diagnostics</a></h3> |
| <p>Some messages are emitted via <a href="diagnostics.html#lints">lints</a>, where the user can control the |
| level. Most diagnostics are hard-coded such that the user cannot control the |
| level.</p> |
| <p>Usually it is obvious whether a diagnostic should be "fixed" or a lint, but |
| there are some grey areas.</p> |
| <p>Here are a few examples:</p> |
| <ul> |
| <li>Borrow checker errors: these are fixed errors. The user cannot adjust the |
| level of these diagnostics to silence the borrow checker.</li> |
| <li>Dead code: this is a lint. While the user probably doesn't want dead code in |
| their crate, making this a hard error would make refactoring and development |
| very painful.</li> |
| <li><a href="diagnostics.html#future-incompatible-lints">future-incompatible lints</a>: |
| these are silenceable lints. |
| It was decided that making them fixed errors would cause too much breakage, |
| so warnings are instead emitted, |
| and will eventually be turned into fixed (hard) errors.</li> |
| </ul> |
| <p>Hard-coded warnings (those using methods like <code>span_warn</code>) should be avoided |
| for normal code, preferring to use lints instead. Some cases, such as warnings |
| with CLI flags, will require the use of hard-coded warnings.</p> |
| <p>See the <code>deny</code> <a href="diagnostics.html#diagnostic-levels">lint level</a> below for guidelines when to |
| use an error-level lint instead of a fixed error.</p> |
| <h2 id="diagnostic-output-style-guide"><a class="header" href="#diagnostic-output-style-guide">Diagnostic output style guide</a></h2> |
| <ul> |
| <li>Write in plain simple English. If your message, when shown on a – possibly |
| small – screen (which hasn't been cleaned for a while), cannot be understood |
| by a normal programmer, who just came out of bed after a night partying, |
| it's too complex.</li> |
| <li><code>Error</code>, <code>Warning</code>, <code>Note</code>, and <code>Help</code> messages start with a lowercase |
| letter and do not end with punctuation.</li> |
| <li>Error messages should be succinct. Users will see these error messages many |
| times, and more verbose descriptions can be viewed with the <code>--explain</code> |
| flag. That said, don't make it so terse that it's hard to understand.</li> |
| <li>The word "illegal" is illegal. Prefer "invalid" or a more specific word |
| instead.</li> |
| <li>Errors should document the span of code where they occur (use |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html"><code>rustc_errors::DiagCtxt</code></a>'s |
| <code>span_*</code> methods or a diagnostic struct's <code>#[primary_span]</code> to easily do |
| this). Also <code>note</code> other spans that have contributed to the error if the span |
| isn't too large.</li> |
| <li>When emitting a message with span, try to reduce the span to the smallest |
| amount possible that still signifies the issue</li> |
| <li>Try not to emit multiple error messages for the same error. This may require |
| detecting duplicates.</li> |
| <li>When the compiler has too little information for a specific error message, |
| consult with the compiler team to add new attributes for library code that |
| allow adding more information. For example see |
| <a href="diagnostics.html#rustc_on_unimplemented"><code>#[rustc_on_unimplemented]</code></a>. Use these |
| annotations when available!</li> |
| <li>Keep in mind that Rust's learning curve is rather steep, and that the |
| compiler messages are an important learning tool.</li> |
| <li>When talking about the compiler, call it <code>the compiler</code>, not <code>Rust</code> or |
| <code>rustc</code>.</li> |
| <li>Use the <a href="https://en.wikipedia.org/wiki/Serial_comma">Oxford comma</a> when |
| writing lists of items.</li> |
| </ul> |
| <h3 id="lint-naming"><a class="header" href="#lint-naming">Lint naming</a></h3> |
| <p>From <a href="https://github.com/rust-lang/rfcs/blob/master/text/0344-conventions-galore.md#lints">RFC 0344</a>, lint names should be consistent, with the following |
| guidelines:</p> |
| <p>The basic rule is: the lint name should make sense when read as "allow |
| <em>lint-name</em>" or "allow <em>lint-name</em> items". For example, "allow |
| <code>deprecated</code> items" and "allow <code>dead_code</code>" makes sense, while "allow |
| <code>unsafe_block</code>" is ungrammatical (should be plural).</p> |
| <ul> |
| <li> |
| <p>Lint names should state the bad thing being checked for, e.g. <code>deprecated</code>, |
| so that <code>#[allow(deprecated)]</code> (items) reads correctly. Thus <code>ctypes</code> is not |
| an appropriate name; <code>improper_ctypes</code> is.</p> |
| </li> |
| <li> |
| <p>Lints that apply to arbitrary items (like the stability lints) should just |
| mention what they check for: use <code>deprecated</code> rather than |
| <code>deprecated_items</code>. This keeps lint names short. (Again, think "allow |
| <em>lint-name</em> items".)</p> |
| </li> |
| <li> |
| <p>If a lint applies to a specific grammatical class, mention that class and |
| use the plural form: use <code>unused_variables</code> rather than <code>unused_variable</code>. |
| This makes <code>#[allow(unused_variables)]</code> read correctly.</p> |
| </li> |
| <li> |
| <p>Lints that catch unnecessary, unused, or useless aspects of code should use |
| the term <code>unused</code>, e.g. <code>unused_imports</code>, <code>unused_typecasts</code>.</p> |
| </li> |
| <li> |
| <p>Use snake case in the same way you would for function names.</p> |
| </li> |
| </ul> |
| <h3 id="diagnostic-levels"><a class="header" href="#diagnostic-levels">Diagnostic levels</a></h3> |
| <p>Guidelines for different diagnostic levels:</p> |
| <ul> |
| <li> |
| <p><code>error</code>: emitted when the compiler detects a problem that makes it unable to |
| compile the program, either because the program is invalid or the programmer |
| has decided to make a specific <code>warning</code> into an error.</p> |
| </li> |
| <li> |
| <p><code>warning</code>: emitted when the compiler detects something odd about a program. |
| Care should be taken when adding warnings to avoid warning fatigue, and |
| avoid false-positives where there really isn't a problem with the code. Some |
| examples of when it is appropriate to issue a warning:</p> |
| <ul> |
| <li>A situation where the user <em>should</em> take action, such as swap out a |
| deprecated item, or use a <code>Result</code>, but otherwise doesn't prevent |
| compilation.</li> |
| <li>Unnecessary syntax that can be removed without affecting the semantics of |
| the code. For example, unused code, or unnecessary <code>unsafe</code>.</li> |
| <li>Code that is very likely to be incorrect, dangerous, or confusing, but the |
| language technically allows, and is not ready or confident enough to make |
| an error. For example <code>unused_comparisons</code> (out of bounds comparisons) or |
| <code>bindings_with_variant_name</code> (the user likely did not intend to create a |
| binding in a pattern).</li> |
| <li><a href="diagnostics.html#future-incompatible">Future-incompatible lints</a>, where something was |
| accidentally or erroneously accepted in the past, but rejecting would |
| cause excessive breakage in the ecosystem.</li> |
| <li>Stylistic choices. For example, camel or snake case, or the <code>dyn</code> trait |
| warning in the 2018 edition. These have a high bar to be added, and should |
| only be used in exceptional circumstances. Other stylistic choices should |
| either be allow-by-default lints, or part of other tools like Clippy or |
| rustfmt.</li> |
| </ul> |
| </li> |
| <li> |
| <p><code>help</code>: emitted following an <code>error</code> or <code>warning</code> to give additional |
| information to the user about how to solve their problem. These messages |
| often include a suggestion string and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/enum.Applicability.html"><code>rustc_errors::Applicability</code></a> |
| confidence level to guide automated source fixes by tools. See the |
| <a href="diagnostics.html#suggestions">Suggestions</a> section for more details.</p> |
| <p>The error or warning portion should <em>not</em> suggest how to fix the problem, |
| only the "help" sub-diagnostic should.</p> |
| </li> |
| <li> |
| <p><code>note</code>: emitted to given more context and identify additional circumstances |
| and parts of the code that caused the warning or error. For example, the |
| borrow checker will note any previous conflicting borrows.</p> |
| <p><code>help</code> vs <code>note</code>: <code>help</code> should be used to show changes the user can |
| possibly make to fix the problem. <code>note</code> should be used for everything else, |
| such as other context, information and facts, online resources to read, etc.</p> |
| </li> |
| </ul> |
| <p>Not to be confused with <em>lint levels</em>, whose guidelines are:</p> |
| <ul> |
| <li> |
| <p><code>forbid</code>: Lints should never default to <code>forbid</code>.</p> |
| </li> |
| <li> |
| <p><code>deny</code>: Equivalent to <code>error</code> diagnostic level. Some examples:</p> |
| <ul> |
| <li>A future-incompatible or edition-based lint that has graduated from the |
| warning level.</li> |
| <li>Something that has an extremely high confidence that is incorrect, but |
| still want an escape hatch to allow it to pass.</li> |
| </ul> |
| </li> |
| <li> |
| <p><code>warn</code>: Equivalent to the <code>warning</code> diagnostic level. See <code>warning</code> above |
| for guidelines.</p> |
| </li> |
| <li> |
| <p><code>allow</code>: Examples of the kinds of lints that should default to <code>allow</code>:</p> |
| <ul> |
| <li>The lint has a too high false positive rate.</li> |
| <li>The lint is too opinionated.</li> |
| <li>The lint is experimental.</li> |
| <li>The lint is used for enforcing something that is not normally enforced. |
| For example, the <code>unsafe_code</code> lint can be used to prevent usage of unsafe |
| code.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>More information about lint levels can be found in the <a href="https://doc.rust-lang.org/nightly/rustc/lints/levels.html">rustc |
| book</a> and the <a href="https://doc.rust-lang.org/nightly/reference/attributes/diagnostics.html#lint-check-attributes">reference</a>.</p> |
| <h2 id="helpful-tips-and-options"><a class="header" href="#helpful-tips-and-options">Helpful tips and options</a></h2> |
| <h3 id="finding-the-source-of-errors"><a class="header" href="#finding-the-source-of-errors">Finding the source of errors</a></h3> |
| <p>There are three main ways to find where a given error is emitted:</p> |
| <ul> |
| <li> |
| <p><code>grep</code> for either a sub-part of the error message/label or error code. This |
| usually works well and is straightforward, but there are some cases where |
| the code emitting the error is removed from the code where the error is |
| constructed behind a relatively deep call-stack. Even then, it is a good way |
| to get your bearings.</p> |
| </li> |
| <li> |
| <p>Invoking <code>rustc</code> with the nightly-only flag <code>-Z treat-err-as-bug=1</code> |
| will treat the first error being emitted as an Internal Compiler Error, which |
| allows you to get a |
| stack trace at the point the error has been emitted. Change the <code>1</code> to |
| something else if you wish to trigger on a later error.</p> |
| <p>There are limitations with this approach:</p> |
| <ul> |
| <li>Some calls get elided from the stack trace because they get inlined in the compiled <code>rustc</code>.</li> |
| <li>The <em>construction</em> of the error is far away from where it is <em>emitted</em>, |
| a problem similar to the one we faced with the <code>grep</code> approach. |
| In some cases, we buffer multiple errors in order to emit them in order.</li> |
| </ul> |
| </li> |
| <li> |
| <p>Invoking <code>rustc</code> with <code>-Z track-diagnostics</code> will print error creation |
| locations alongside the error.</p> |
| </li> |
| </ul> |
| <p>The regular development practices apply: judicious use of <code>debug!()</code> statements |
| and use of a debugger to trigger break points in order to figure out in what |
| order things are happening.</p> |
| <h2 id="span"><a class="header" href="#span"><code>Span</code></a></h2> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html"><code>Span</code></a> is the primary data structure in <code>rustc</code> used to represent a |
| location in the code being compiled. <code>Span</code>s are attached to most constructs in |
| HIR and MIR, allowing for more informative error reporting.</p> |
| <p>A <code>Span</code> can be looked up in a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html"><code>SourceMap</code></a> to get a "snippet" |
| useful for displaying errors with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html#method.span_to_snippet"><code>span_to_snippet</code></a> and other |
| similar methods on the <code>SourceMap</code>.</p> |
| <h2 id="error-messages"><a class="header" href="#error-messages">Error messages</a></h2> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/index.html"><code>rustc_errors</code></a> crate defines most of the utilities used for |
| reporting errors.</p> |
| <p>Diagnostics can be implemented as types which implement the <code>Diagnostic</code> |
| trait. This is preferred for new diagnostics as it enforces a separation |
| between diagnostic emitting logic and the main code paths. For less-complex |
| diagnostics, the <code>Diagnostic</code> trait can be derived -- see <a href="./diagnostics/diagnostic-structs.html">Diagnostic |
| structs</a>. Within the trait implementation, the APIs |
| described below can be used as normal.</p> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html"><code>DiagCtxt</code></a> has methods that create and emit errors. These methods |
| usually have names like <code>span_err</code> or <code>struct_span_err</code> or <code>span_warn</code>, etc... |
| There are lots of them; they emit different types of "errors", such as |
| warnings, errors, fatal errors, suggestions, etc.</p> |
| <p>In general, there are two classes of such methods: ones that emit an error |
| directly and ones that allow finer control over what to emit. For example, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html#method.span_err"><code>span_err</code></a> emits the given error message at the given <code>Span</code>, but |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html#method.struct_span_err"><code>struct_span_err</code></a> instead returns a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html"><code>Diag</code></a>.</p> |
| <p>Most of these methods will accept strings, but it is recommended that typed |
| identifiers for translatable diagnostics be used for new diagnostics (see |
| <a href="./diagnostics/translation.html">Translation</a>).</p> |
| <p><code>Diag</code> allows you to add related notes and suggestions to an error |
| before emitting it by calling the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html#method.emit"><code>emit</code></a> method. (Failing to either |
| emit or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html#method.cancel">cancel</a> a <code>Diag</code> will result in an ICE.) See the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html">docs</a> for more info on what you can do.</p> |
| <pre><code class="language-rust ignore">// Get a `Diag`. This does _not_ emit an error yet. |
| let mut err = sess.dcx.struct_span_err(sp, fluent::example::example_error); |
| |
| // In some cases, you might need to check if `sp` is generated by a macro to |
| // avoid printing weird errors about macro-generated code. |
| |
| if let Ok(snippet) = sess.source_map().span_to_snippet(sp) { |
| // Use the snippet to generate a suggested fix |
| err.span_suggestion(suggestion_sp, fluent::example::try_qux_suggestion, format!("qux {}", snippet)); |
| } else { |
| // If we weren't able to generate a snippet, then emit a "help" message |
| // instead of a concrete "suggestion". In practice this is unlikely to be |
| // reached. |
| err.span_help(suggestion_sp, fluent::example::qux_suggestion); |
| } |
| |
| // emit the error |
| err.emit();</code></pre> |
| <pre><code class="language-fluent">example-example-error = oh no! this is an error! |
| .try-qux-suggestion = try using a qux here |
| .qux-suggestion = you could use a qux here instead |
| </code></pre> |
| <h2 id="suggestions"><a class="header" href="#suggestions">Suggestions</a></h2> |
| <p>In addition to telling the user exactly <em>why</em> their code is wrong, it's |
| oftentimes furthermore possible to tell them how to fix it. To this end, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html"><code>Diag</code></a> offers a structured suggestions API, which formats code |
| suggestions pleasingly in the terminal, or (when the <code>--error-format json</code> flag |
| is passed) as JSON for consumption by tools like <a href="https://github.com/rust-lang/rustfix"><code>rustfix</code></a>.</p> |
| <p>Not all suggestions should be applied mechanically, they have a degree of |
| confidence in the suggested code, from high |
| (<code>Applicability::MachineApplicable</code>) to low (<code>Applicability::MaybeIncorrect</code>). |
| Be conservative when choosing the level. Use the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html#method.span_suggestion"><code>span_suggestion</code></a> method of <code>Diag</code> to |
| make a suggestion. The last argument provides a hint to tools whether |
| the suggestion is mechanically applicable or not.</p> |
| <p>Suggestions point to one or more spans with corresponding code that will |
| replace their current content.</p> |
| <p>The message that accompanies them should be understandable in the following |
| contexts:</p> |
| <ul> |
| <li>shown as an independent sub-diagnostic (this is the default output)</li> |
| <li>shown as a label pointing at the affected span (this is done automatically if |
| some heuristics for verbosity are met)</li> |
| <li>shown as a <code>help</code> sub-diagnostic with no content (used for cases where the |
| suggestion is obvious from the text, but we still want to let tools to apply |
| them)</li> |
| <li>not shown (used for <em>very</em> obvious cases, but we still want to allow tools to |
| apply them)</li> |
| </ul> |
| <p>For example, to make our <code>qux</code> suggestion machine-applicable, we would do:</p> |
| <pre><code class="language-rust ignore">let mut err = sess.dcx.struct_span_err(sp, fluent::example::message); |
| |
| if let Ok(snippet) = sess.source_map().span_to_snippet(sp) { |
| err.span_suggestion( |
| suggestion_sp, |
| fluent::example::try_qux_suggestion, |
| format!("qux {}", snippet), |
| Applicability::MachineApplicable, |
| ); |
| } else { |
| err.span_help(suggestion_sp, fluent::example::qux_suggestion); |
| } |
| |
| err.emit();</code></pre> |
| <p>This might emit an error like</p> |
| <pre><code class="language-console">$ rustc mycode.rs |
| error[E0999]: oh no! this is an error! |
| --> mycode.rs:3:5 |
| | |
| 3 | sad() |
| | ^ help: try using a qux here: `qux sad()` |
| |
| error: aborting due to previous error |
| |
| For more information about this error, try `rustc --explain E0999`. |
| </code></pre> |
| <p>In some cases, like when the suggestion spans multiple lines or when there are |
| multiple suggestions, the suggestions are displayed on their own:</p> |
| <pre><code class="language-console">error[E0999]: oh no! this is an error! |
| --> mycode.rs:3:5 |
| | |
| 3 | sad() |
| | ^ |
| help: try using a qux here: |
| | |
| 3 | qux sad() |
| | ^^^ |
| |
| error: aborting due to previous error |
| |
| For more information about this error, try `rustc --explain E0999`. |
| </code></pre> |
| <p>The possible values of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/enum.Applicability.html"><code>Applicability</code></a> are:</p> |
| <ul> |
| <li><code>MachineApplicable</code>: Can be applied mechanically.</li> |
| <li><code>HasPlaceholders</code>: Cannot be applied mechanically because it has placeholder |
| text in the suggestions. For example: <code>try adding a type: `let x: <type>` </code>.</li> |
| <li><code>MaybeIncorrect</code>: Cannot be applied mechanically because the suggestion may |
| or may not be a good one.</li> |
| <li><code>Unspecified</code>: Cannot be applied mechanically because we don't know which |
| of the above cases it falls into.</li> |
| </ul> |
| <h3 id="suggestion-style-guide"><a class="header" href="#suggestion-style-guide">Suggestion Style Guide</a></h3> |
| <ul> |
| <li> |
| <p>Suggestions should not be a question. In particular, language like "did you |
| mean" should be avoided. Sometimes, it's unclear why a particular suggestion |
| is being made. In these cases, it's better to be upfront about what the |
| suggestion is.</p> |
| <p>Compare "did you mean: <code>Foo</code>" vs. "there is a struct with a similar name: <code>Foo</code>".</p> |
| </li> |
| <li> |
| <p>The message should not contain any phrases like "the following", "as shown", |
| etc. Use the span to convey what is being talked about.</p> |
| </li> |
| <li> |
| <p>The message may contain further instruction such as "to do xyz, use" or "to do |
| xyz, use abc".</p> |
| </li> |
| <li> |
| <p>The message may contain a name of a function, variable, or type, but avoid |
| whole expressions.</p> |
| </li> |
| </ul> |
| <h2 id="lints-1"><a class="header" href="#lints-1">Lints</a></h2> |
| <p>The compiler linting infrastructure is defined in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/lint/index.html"><code>rustc_middle::lint</code></a> |
| module.</p> |
| <h3 id="when-do-lints-run"><a class="header" href="#when-do-lints-run">When do lints run?</a></h3> |
| <p>Different lints will run at different times based on what information the lint |
| needs to do its job. Some lints get grouped into <em>passes</em> where the lints |
| within a pass are processed together via a single visitor. Some of the passes |
| are:</p> |
| <ul> |
| <li> |
| <p>Pre-expansion pass: Works on <a href="the-parser.html">AST nodes</a> before <a href="macro-expansion.html">macro expansion</a>. This |
| should generally be avoided.</p> |
| <ul> |
| <li>Example: <a href="https://doc.rust-lang.org/rustc/lints/listing/allowed-by-default.html#keyword-idents"><code>keyword_idents</code></a> checks for identifiers that will become |
| keywords in future editions, but is sensitive to identifiers used in |
| macros.</li> |
| </ul> |
| </li> |
| <li> |
| <p>Early lint pass: Works on <a href="the-parser.html">AST nodes</a> after <a href="macro-expansion.html">macro expansion</a> and name |
| resolution, just before <a href="./hir/lowering.html">AST lowering</a>. These lints are for purely |
| syntactical lints.</p> |
| <ul> |
| <li>Example: The <a href="https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#unused-parens"><code>unused_parens</code></a> lint checks for parenthesized-expressions |
| in situations where they are not needed, like an <code>if</code> condition.</li> |
| </ul> |
| </li> |
| <li> |
| <p>Late lint pass: Works on <a href="hir.html">HIR nodes</a>, towards the end of <a href="part-4-intro.html">analysis</a> (after |
| borrow checking, etc.). These lints have full type information available. |
| Most lints are late.</p> |
| <ul> |
| <li>Example: The <a href="https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#invalid-value"><code>invalid_value</code></a> lint (which checks for obviously invalid |
| uninitialized values) is a late lint because it needs type information to |
| figure out whether a type allows being left uninitialized.</li> |
| </ul> |
| </li> |
| <li> |
| <p>MIR pass: Works on <a href="mir/index.html">MIR nodes</a>. This isn't quite the same as other passes; |
| lints that work on MIR nodes have their own methods for running.</p> |
| <ul> |
| <li>Example: The <a href="https://doc.rust-lang.org/rustc/lints/listing/deny-by-default.html#arithmetic-overflow"><code>arithmetic_overflow</code></a> lint is emitted when it detects a |
| constant value that may overflow.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Most lints work well via the pass systems, and they have a fairly |
| straightforward interface and easy way to integrate (mostly just implementing |
| a specific <code>check</code> function). However, some lints are easier to write when |
| they live on a specific code path anywhere in the compiler. For example, the |
| <a href="https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#unused-mut"><code>unused_mut</code></a> lint is implemented in the borrow checker as it requires some |
| information and state in the borrow checker.</p> |
| <p>Some of these inline lints fire before the linting system is ready. Those |
| lints will be <em>buffered</em> where they are held until later phases of the |
| compiler when the linting system is ready. See <a href="diagnostics.html#linting-early-in-the-compiler">Linting early in the |
| compiler</a>.</p> |
| <h3 id="lint-definition-terms"><a class="header" href="#lint-definition-terms">Lint definition terms</a></h3> |
| <p>Lints are managed via the <a href="diagnostics/lintstore.html"><code>LintStore</code></a> and get registered in |
| various ways. The following terms refer to the different classes of lints |
| generally based on how they are registered.</p> |
| <ul> |
| <li><em>Built-in</em> lints are defined inside the compiler source.</li> |
| <li><em>Driver-registered</em> lints are registered when the compiler driver is created |
| by an external driver. This is the mechanism used by Clippy, for example.</li> |
| <li><em>Tool</em> lints are lints with a path prefix like <code>clippy::</code> or <code>rustdoc::</code>.</li> |
| <li><em>Internal</em> lints are the <code>rustc::</code> scoped tool lints that only run on the |
| rustc source tree itself and are defined in the compiler source like a |
| regular built-in lint.</li> |
| </ul> |
| <p>More information about lint registration can be found in the <a href="diagnostics/lintstore.html">LintStore</a> |
| chapter.</p> |
| <h3 id="declaring-a-lint"><a class="header" href="#declaring-a-lint">Declaring a lint</a></h3> |
| <p>The built-in compiler lints are defined in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/index.html"><code>rustc_lint</code></a> |
| crate. Lints that need to be implemented in other crates are defined in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/index.html"><code>rustc_lint_defs</code></a>. You should prefer to place lints in <code>rustc_lint</code> if |
| possible. One benefit is that it is close to the dependency root, so it can be |
| much faster to work on.</p> |
| <p>Every lint is implemented via a <code>struct</code> that implements the <code>LintPass</code> <code>trait</code> |
| (you can also implement one of the more specific lint pass traits, either |
| <code>EarlyLintPass</code> or <code>LateLintPass</code> depending on when is best for your lint to run). |
| The trait implementation allows you to check certain syntactic constructs |
| as the linter walks the AST. You can then choose to emit lints in a |
| very similar way to compile errors.</p> |
| <p>You also declare the metadata of a particular lint via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/macro.declare_lint.html"><code>declare_lint!</code></a> |
| macro. This macro includes the name, the default level, a short description, and some |
| more details.</p> |
| <p>Note that the lint and the lint pass must be registered with the compiler.</p> |
| <p>For example, the following lint checks for uses |
| of <code>while true { ... }</code> and suggests using <code>loop { ... }</code> instead.</p> |
| <pre><code class="language-rust ignore">// Declare a lint called `WHILE_TRUE` |
| declare_lint! { |
| WHILE_TRUE, |
| |
| // warn-by-default |
| Warn, |
| |
| // This string is the lint description |
| "suggest using `loop { }` instead of `while true { }`" |
| } |
| |
| // This declares a struct and a lint pass, providing a list of associated lints. The |
| // compiler currently doesn't use the associated lints directly (e.g., to not |
| // run the pass or otherwise check that the pass emits the appropriate set of |
| // lints). However, it's good to be accurate here as it's possible that we're |
| // going to register the lints via the get_lints method on our lint pass (that |
| // this macro generates). |
| declare_lint_pass!(WhileTrue => [WHILE_TRUE]); |
| |
| // Helper function for `WhileTrue` lint. |
| // Traverse through any amount of parenthesis and return the first non-parens expression. |
| fn pierce_parens(mut expr: &ast::Expr) -> &ast::Expr { |
| while let ast::ExprKind::Paren(sub) = &expr.kind { |
| expr = sub; |
| } |
| expr |
| } |
| |
| // `EarlyLintPass` has lots of methods. We only override the definition of |
| // `check_expr` for this lint because that's all we need, but you could |
| // override other methods for your own lint. See the rustc docs for a full |
| // list of methods. |
| impl EarlyLintPass for WhileTrue { |
| fn check_expr(&mut self, cx: &EarlyContext<'_>, e: &ast::Expr) { |
| if let ast::ExprKind::While(cond, ..) = &e.kind |
| && let ast::ExprKind::Lit(ref lit) = pierce_parens(cond).kind |
| && let ast::LitKind::Bool(true) = lit.kind |
| && !lit.span.from_expansion() |
| { |
| let condition_span = cx.sess.source_map().guess_head_span(e.span); |
| cx.struct_span_lint(WHILE_TRUE, condition_span, |lint| { |
| lint.build(fluent::example::use_loop) |
| .span_suggestion_short( |
| condition_span, |
| fluent::example::suggestion, |
| "loop".to_owned(), |
| Applicability::MachineApplicable, |
| ) |
| .emit(); |
| }) |
| } |
| } |
| }</code></pre> |
| <pre><code class="language-fluent">example-use-loop = denote infinite loops with `loop {"{"} ... {"}"}` |
| .suggestion = use `loop` |
| </code></pre> |
| <h3 id="edition-gated-lints"><a class="header" href="#edition-gated-lints">Edition-gated lints</a></h3> |
| <p>Sometimes we want to change the behavior of a lint in a new edition. To do this, |
| we just add the transition to our invocation of <code>declare_lint!</code>:</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| pub ANONYMOUS_PARAMETERS, |
| Allow, |
| "detects anonymous parameters", |
| Edition::Edition2018 => Warn, |
| }</code></pre> |
| <p>This makes the <code>ANONYMOUS_PARAMETERS</code> lint allow-by-default in the 2015 edition |
| but warn-by-default in the 2018 edition.</p> |
| <p>See <a href="./guides/editions.html#edition-specific-lints">Edition-specific lints</a> for more information.</p> |
| <h3 id="feature-gated-lints"><a class="header" href="#feature-gated-lints">Feature-gated lints</a></h3> |
| <p>Lints belonging to a feature should only be usable if the feature is enabled in the |
| crate. To support this, lint declarations can contain a feature gate like so:</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| pub SOME_LINT_NAME, |
| Warn, |
| "a new and useful, but feature gated lint", |
| @feature_gate = sym::feature_name; |
| }</code></pre> |
| <h3 id="future-incompatible-lints"><a class="header" href="#future-incompatible-lints">Future-incompatible lints</a></h3> |
| <p>The use of the term <code>future-incompatible</code> within the compiler has a slightly |
| broader meaning than what rustc exposes to users of the compiler.</p> |
| <p>Inside rustc, future-incompatible lints are for signalling to the user that code they have |
| written may not compile in the future. In general, future-incompatible code |
| exists for two reasons:</p> |
| <ul> |
| <li>The user has written unsound code that the compiler mistakenly accepted. While |
| it is within Rust's backwards compatibility guarantees to fix the soundness hole |
| (breaking the user's code), the lint is there to warn the user that this will happen |
| in some upcoming version of rustc <em>regardless of which edition the code uses</em>. This is the |
| meaning that rustc exclusively exposes to users as "future incompatible".</li> |
| <li>The user has written code that will either no longer compiler <em>or</em> will change |
| meaning in an upcoming <em>edition</em>. These are often called "edition lints" and can be |
| typically seen in the various "edition compatibility" lint groups (e.g., <code>rust_2021_compatibility</code>) |
| that are used to lint against code that will break if the user updates the crate's edition. |
| See <a href="guides/editions.html#migration-lints">migration lints</a> for more details.</li> |
| </ul> |
| <p>A future-incompatible lint should be declared with the <code>@future_incompatible</code> |
| additional "field":</p> |
| <pre><code class="language-rust ignore">declare_lint! { |
| pub ANONYMOUS_PARAMETERS, |
| Allow, |
| "detects anonymous parameters", |
| @future_incompatible = FutureIncompatibleInfo { |
| reference: "issue #41686 <https://github.com/rust-lang/rust/issues/41686>", |
| reason: FutureIncompatibilityReason::EditionError(Edition::Edition2018), |
| }; |
| }</code></pre> |
| <p>Notice the <code>reason</code> field which describes why the future incompatible change is happening. |
| This will change the diagnostic message the user receives as well as determine which |
| lint groups the lint is added to. In the example above, the lint is an "edition lint" |
| (since its "reason" is <code>EditionError</code>), signifying to the user that the use of anonymous |
| parameters will no longer compile in Rust 2018 and beyond.</p> |
| <p>Inside <a href="https://github.com/rust-lang/rust/blob/51fd129ac12d5bfeca7d216c47b0e337bf13e0c2/compiler/rustc_lint/src/context.rs#L212-L237">LintStore::register_lints</a>, lints with <code>future_incompatible</code> |
| fields get placed into either edition-based lint groups (if their <code>reason</code> is tied to |
| an edition) or into the <code>future_incompatibility</code> lint group.</p> |
| <p>If you need a combination of options that's not supported by the |
| <code>declare_lint!</code> macro, you can always change the <code>declare_lint!</code> macro |
| to support this.</p> |
| <h3 id="renaming-or-removing-a-lint"><a class="header" href="#renaming-or-removing-a-lint">Renaming or removing a lint</a></h3> |
| <p>If it is determined that a lint is either improperly named or no longer needed, |
| the lint must be registered for renaming or removal, which will trigger a warning if a user tries |
| to use the old lint name. To declare a rename/remove, add a line with |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LintStore.html#method.register_renamed"><code>store.register_renamed</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LintStore.html#method.register_removed"><code>store.register_removed</code></a> to the code of the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.register_builtins.html"><code>rustc_lint::register_builtins</code></a> function.</p> |
| <pre><code class="language-rust ignore">store.register_renamed("single_use_lifetime", "single_use_lifetimes");</code></pre> |
| <h3 id="lint-groups"><a class="header" href="#lint-groups">Lint groups</a></h3> |
| <p>Lints can be turned on in groups. These groups are declared in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.register_builtins.html"><code>register_builtins</code></a> function in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/index.html"><code>rustc_lint::lib</code></a>. The |
| <code>add_lint_group!</code> macro is used to declare a new group.</p> |
| <p>For example,</p> |
| <pre><code class="language-rust ignore">add_lint_group!(sess, |
| "nonstandard_style", |
| NON_CAMEL_CASE_TYPES, |
| NON_SNAKE_CASE, |
| NON_UPPER_CASE_GLOBALS);</code></pre> |
| <p>This defines the <code>nonstandard_style</code> group which turns on the listed lints. A |
| user can turn on these lints with a <code>!#[warn(nonstandard_style)]</code> attribute in |
| the source code, or by passing <code>-W nonstandard-style</code> on the command line.</p> |
| <p>Some lint groups are created automatically in <code>LintStore::register_lints</code>. For instance, |
| any lint declared with <code>FutureIncompatibleInfo</code> where the reason is |
| <code>FutureIncompatibilityReason::FutureReleaseError</code> (the default when |
| <code>@future_incompatible</code> is used in <code>declare_lint!</code>), will be added to |
| the <code>future_incompatible</code> lint group. Editions also have their own lint groups |
| (e.g., <code>rust_2021_compatibility</code>) automatically generated for any lints signaling |
| future-incompatible code that will break in the specified edition.</p> |
| <h3 id="linting-early-in-the-compiler"><a class="header" href="#linting-early-in-the-compiler">Linting early in the compiler</a></h3> |
| <p>On occasion, you may need to define a lint that runs before the linting system |
| has been initialized (e.g. during parsing or macro expansion). This is |
| problematic because we need to have computed lint levels to know whether we |
| should emit a warning or an error or nothing at all.</p> |
| <p>To solve this problem, we buffer the lints until the linting system is |
| processed. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html#method.buffer_lint"><code>Session</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html#method.buffer_lint"><code>ParseSess</code></a> both have |
| <code>buffer_lint</code> methods that allow you to buffer a lint for later. The linting |
| system automatically takes care of handling buffered lints later.</p> |
| <p>Thus, to define a lint that runs early in the compilation, one defines a lint |
| like normal but invokes the lint with <code>buffer_lint</code>.</p> |
| <h4 id="linting-even-earlier-in-the-compiler"><a class="header" href="#linting-even-earlier-in-the-compiler">Linting even earlier in the compiler</a></h4> |
| <p>The parser (<code>rustc_ast</code>) is interesting in that it cannot have dependencies on |
| any of the other <code>rustc*</code> crates. In particular, it cannot depend on |
| <code>rustc_middle::lint</code> or <code>rustc_lint</code>, where all of the compiler linting |
| infrastructure is defined. That's troublesome!</p> |
| <p>To solve this, <code>rustc_ast</code> defines its own buffered lint type, which |
| <code>ParseSess::buffer_lint</code> uses. After macro expansion, these buffered lints are |
| then dumped into the <code>Session::buffered_lints</code> used by the rest of the compiler.</p> |
| <h2 id="json-diagnostic-output"><a class="header" href="#json-diagnostic-output">JSON diagnostic output</a></h2> |
| <p>The compiler accepts an <code>--error-format json</code> flag to output |
| diagnostics as JSON objects (for the benefit of tools such as <code>cargo fix</code>). It looks like this:</p> |
| <pre><code class="language-console">$ rustc json_error_demo.rs --error-format json |
| {"message":"cannot add `&str` to `{integer}`","code":{"code":"E0277","explanation":"\nYou tried to use a type which doesn't implement some trait in a place which\nexpected that trait. Erroneous code example:\n\n```compile_fail,E0277\n// here we declare the Foo trait with a bar method\ntrait Foo {\n fn bar(&self);\n}\n\n// we now declare a function which takes an object implementing the Foo trait\nfn some_func<T: Foo>(foo: T) {\n foo.bar();\n}\n\nfn main() {\n // we now call the method with the i32 type, which doesn't implement\n // the Foo trait\n some_func(5i32); // error: the trait bound `i32 : Foo` is not satisfied\n}\n```\n\nIn order to fix this error, verify that the type you're using does implement\nthe trait. Example:\n\n```\ntrait Foo {\n fn bar(&self);\n}\n\nfn some_func<T: Foo>(foo: T) {\n foo.bar(); // we can now use this method since i32 implements the\n // Foo trait\n}\n\n// we implement the trait on the i32 type\nimpl Foo for i32 {\n fn bar(&self) {}\n}\n\nfn main() {\n some_func(5i32); // ok!\n}\n```\n\nOr in a generic context, an erroneous code example would look like:\n\n```compile_fail,E0277\nfn some_func<T>(foo: T) {\n println!(\"{:?}\", foo); // error: the trait `core::fmt::Debug` is not\n // implemented for the type `T`\n}\n\nfn main() {\n // We now call the method with the i32 type,\n // which *does* implement the Debug trait.\n some_func(5i32);\n}\n```\n\nNote that the error here is in the definition of the generic function: Although\nwe only call it with a parameter that does implement `Debug`, the compiler\nstill rejects the function: It must work with all possible input types. In\norder to make this example compile, we need to restrict the generic type we're\naccepting:\n\n```\nuse std::fmt;\n\n// Restrict the input type to types that implement Debug.\nfn some_func<T: fmt::Debug>(foo: T) {\n println!(\"{:?}\", foo);\n}\n\nfn main() {\n // Calling the method is still fine, as i32 implements Debug.\n some_func(5i32);\n\n // This would fail to compile now:\n // struct WithoutDebug;\n // some_func(WithoutDebug);\n}\n```\n\nRust only looks at the signature of the called function, as such it must\nalready specify all requirements that will be used for every type parameter.\n"},"level":"error","spans":[{"file_name":"json_error_demo.rs","byte_start":50,"byte_end":51,"line_start":4,"line_end":4,"column_start":7,"column_end":8,"is_primary":true,"text":[{"text":" a + b","highlight_start":7,"highlight_end":8}],"label":"no implementation for `{integer} + &str`","suggested_replacement":null,"suggestion_applicability":null,"expansion":null}],"children":[{"message":"the trait `std::ops::Add<&str>` is not implemented for `{integer}`","code":null,"level":"help","spans":[],"children":[],"rendered":null}],"rendered":"error[E0277]: cannot add `&str` to `{integer}`\n --> json_error_demo.rs:4:7\n |\n4 | a + b\n | ^ no implementation for `{integer} + &str`\n |\n = help: the trait `std::ops::Add<&str>` is not implemented for `{integer}`\n\n"} |
| {"message":"aborting due to previous error","code":null,"level":"error","spans":[],"children":[],"rendered":"error: aborting due to previous error\n\n"} |
| {"message":"For more information about this error, try `rustc --explain E0277`.","code":null,"level":"","spans":[],"children":[],"rendered":"For more information about this error, try `rustc --explain E0277`.\n"} |
| </code></pre> |
| <p>Note that the output is a series of lines, each of which is a JSON |
| object, but the series of lines taken together is, unfortunately, not |
| valid JSON, thwarting tools and tricks (such as <a href="https://docs.python.org/3/library/json.html#module-json.tool">piping to <code>python3 -m json.tool</code></a>) |
| that require such. (One speculates that this was intentional for LSP |
| performance purposes, so that each line/object can be sent as |
| it is flushed?)</p> |
| <p>Also note the "rendered" field, which contains the "human" output as a |
| string; this was introduced so that UI tests could both make use of |
| the structured JSON and see the "human" output (well, <em>sans</em> colors) |
| without having to compile everything twice.</p> |
| <p>The "human" readable and the json format emitter can be found under |
| <code>rustc_errors</code>, both were moved from the <code>rustc_ast</code> crate to the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/index.html">rustc_errors crate</a>.</p> |
| <p>The JSON emitter defines <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/json/struct.Diagnostic.html">its own <code>Diagnostic</code> |
| struct</a> |
| (and sub-structs) for the JSON serialization. Don't confuse this with |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html"><code>errors::Diag</code></a>!</p> |
| <h2 id="rustc_on_unimplemented"><a class="header" href="#rustc_on_unimplemented"><code>#[rustc_on_unimplemented]</code></a></h2> |
| <p>This attribute allows trait definitions to modify error messages when an implementation was |
| expected but not found. The string literals in the attribute are format strings and can be |
| formatted with named parameters. See the Formatting |
| section below for what parameters are permitted.</p> |
| <pre><code class="language-rust ignore">#[rustc_on_unimplemented(message = "an iterator over \ |
| elements of type `{A}` cannot be built from a \ |
| collection of type `{Self}`")] |
| trait MyIterator<A> { |
| fn next(&mut self) -> A; |
| } |
| |
| fn iterate_chars<I: MyIterator<char>>(i: I) { |
| // ... |
| } |
| |
| fn main() { |
| iterate_chars(&[1, 2, 3][..]); |
| }</code></pre> |
| <p>When the user compiles this, they will see the following;</p> |
| <pre><code class="language-txt">error[E0277]: an iterator over elements of type `char` cannot be built from a collection of type `&[{integer}]` |
| --> src/main.rs:13:19 |
| | |
| 13 | iterate_chars(&[1, 2, 3][..]); |
| | ------------- ^^^^^^^^^^^^^^ the trait `MyIterator<char>` is not implemented for `&[{integer}]` |
| | | |
| | required by a bound introduced by this call |
| | |
| note: required by a bound in `iterate_chars` |
| </code></pre> |
| <p>You can modify the contents of:</p> |
| <ul> |
| <li>the main error message (<code>message</code>)</li> |
| <li>the label (<code>label</code>)</li> |
| <li>the note(s) (<code>note</code>)</li> |
| </ul> |
| <p>For example, the following attribute</p> |
| <pre><code class="language-rust ignore">#[rustc_on_unimplemented(message = "message", label = "label", note = "note")] |
| trait MyIterator<A> { |
| fn next(&mut self) -> A; |
| }</code></pre> |
| <p>Would generate the following output:</p> |
| <pre><code class="language-text">error[E0277]: message |
| --> <file>:10:19 |
| | |
| 10 | iterate_chars(&[1, 2, 3][..]); |
| | ------------- ^^^^^^^^^^^^^^ label |
| | | |
| | required by a bound introduced by this call |
| | |
| = help: the trait `MyIterator<char>` is not implemented for `&[{integer}]` |
| = note: note |
| note: required by a bound in `iterate_chars` |
| </code></pre> |
| <p>The functionality discussed so far is also available with |
| <a href="https://doc.rust-lang.org/nightly/reference/attributes/diagnostics.html#the-diagnosticon_unimplemented-attribute"><code>#[diagnostic::on_unimplemented]</code></a>. |
| If you can, you should use that instead.</p> |
| <h3 id="filtering"><a class="header" href="#filtering">Filtering</a></h3> |
| <p>To allow more targeted error messages, it is possible to filter the |
| application of these fields with <code>on</code>.</p> |
| <p>You can filter on the following boolean flags:</p> |
| <ul> |
| <li><code>crate_local</code>: whether the code causing the trait bound to not be |
| fulfilled is part of the user's crate. This is used to avoid suggesting |
| code changes that would require modifying a dependency.</li> |
| <li><code>direct</code>: whether this is an user-specified rather than derived obligation.</li> |
| <li><code>from_desugaring</code>: whether we are in some kind of desugaring, like <code>?</code> |
| or a <code>try</code> block for example. This flag can also be matched on, see below.</li> |
| </ul> |
| <p>You can match on the following names and values, using <code>name = "value"</code>:</p> |
| <ul> |
| <li><code>cause</code>: Match against one variant of the <code>ObligationCauseCode</code> |
| enum. Only <code>"MainFunctionType"</code> is supported.</li> |
| <li><code>from_desugaring</code>: Match against a particular variant of the <code>DesugaringKind</code> |
| enum. The desugaring is identified by its variant name, for example |
| <code>"QuestionMark"</code> for <code>?</code> desugaring or <code>"TryBlock"</code> for <code>try</code> blocks.</li> |
| <li><code>Self</code> and any generic arguments of the trait, like <code>Self = "alloc::string::String"</code> |
| or <code>Rhs="i32"</code>.</li> |
| </ul> |
| <p>The compiler can provide several values to match on, for example:</p> |
| <ul> |
| <li>the self_ty, pretty printed with and without type arguments resolved.</li> |
| <li><code>"{integral}"</code>, if self_ty is an integral of which the type is known.</li> |
| <li><code>"[]"</code>, <code>"[{ty}]"</code>, <code>"[{ty}; _]"</code>, <code>"[{ty}; $N]"</code> when applicable.</li> |
| <li>references to said slices and arrays.</li> |
| <li><code>"fn"</code>, <code>"unsafe fn"</code> or <code>"#[target_feature] fn"</code> when self is a function.</li> |
| <li><code>"{integer}"</code> and <code>"{float}"</code> if the type is a number but we haven't inferred it yet.</li> |
| <li>combinations of the above, like <code>"[{integral}; _]"</code>.</li> |
| </ul> |
| <p>For example, the <code>Iterator</code> trait can be filtered in the following way:</p> |
| <pre><code class="language-rust ignore">#[rustc_on_unimplemented( |
| on(Self = "&str", note = "call `.chars()` or `.as_bytes()` on `{Self}`"), |
| message = "`{Self}` is not an iterator", |
| label = "`{Self}` is not an iterator", |
| note = "maybe try calling `.iter()` or a similar method" |
| )] |
| pub trait Iterator {}</code></pre> |
| <p>Which would produce the following outputs:</p> |
| <pre><code class="language-text">error[E0277]: `Foo` is not an iterator |
| --> src/main.rs:4:16 |
| | |
| 4 | for foo in Foo {} |
| | ^^^ `Foo` is not an iterator |
| | |
| = note: maybe try calling `.iter()` or a similar method |
| = help: the trait `std::iter::Iterator` is not implemented for `Foo` |
| = note: required by `std::iter::IntoIterator::into_iter` |
| |
| error[E0277]: `&str` is not an iterator |
| --> src/main.rs:5:16 |
| | |
| 5 | for foo in "" {} |
| | ^^ `&str` is not an iterator |
| | |
| = note: call `.chars()` or `.bytes() on `&str` |
| = help: the trait `std::iter::Iterator` is not implemented for `&str` |
| = note: required by `std::iter::IntoIterator::into_iter` |
| </code></pre> |
| <p>The <code>on</code> filter accepts <code>all</code>, <code>any</code> and <code>not</code> predicates similar to the <code>cfg</code> attribute:</p> |
| <pre><code class="language-rust ignore">#[rustc_on_unimplemented(on( |
| all(Self = "&str", T = "alloc::string::String"), |
| note = "you can coerce a `{T}` into a `{Self}` by writing `&*variable`" |
| ))] |
| pub trait From<T>: Sized { |
| /* ... */ |
| }</code></pre> |
| <h3 id="formatting-1"><a class="header" href="#formatting-1">Formatting</a></h3> |
| <p>The string literals are format strings that accept parameters wrapped in braces |
| but positional and listed parameters and format specifiers are not accepted. |
| The following parameter names are valid:</p> |
| <ul> |
| <li><code>Self</code> and all generic parameters of the trait.</li> |
| <li><code>This</code>: the name of the trait the attribute is on, without generics.</li> |
| <li><code>Trait</code>: the name of the "sugared" trait. See <code>TraitRefPrintSugared</code>.</li> |
| <li><code>ItemContext</code>: the kind of <code>hir::Node</code> we're in, things like <code>"an async block"</code>, |
| <code>"a function"</code>, <code>"an async function"</code>, etc.</li> |
| </ul> |
| <p>Something like:</p> |
| <pre><code class="language-rust ignore">#![feature(rustc_attrs)] |
| |
| #[rustc_on_unimplemented(message = "Self = `{Self}`, \ |
| T = `{T}`, this = `{This}`, trait = `{Trait}`, \ |
| context = `{ItemContext}`")] |
| pub trait From<T>: Sized { |
| fn from(x: T) -> Self; |
| } |
| |
| fn main() { |
| let x: i8 = From::from(42_i32); |
| }</code></pre> |
| <p>Will format the message into</p> |
| <pre><code class="language-text">"Self = `i8`, T = `i32`, this = `From`, trait = `From<i32>`, context = `a function`" |
| </code></pre> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-estebank"> |
| <p>This rule of thumb was suggested by <strong>@estebank</strong> <a href="https://github.com/rust-lang/rustc-dev-guide/pull/967#issuecomment-733218283">here</a>. <a href="#fr-estebank-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="diagnostic-and-subdiagnostic-structs"><a class="header" href="#diagnostic-and-subdiagnostic-structs">Diagnostic and subdiagnostic structs</a></h1> |
| <p>rustc has three diagnostic traits that can be used to create diagnostics: |
| <code>Diagnostic</code>, <code>LintDiagnostic</code>, and <code>Subdiagnostic</code>.</p> |
| <p>For simple diagnostics, |
| derived impls can be used, e.g. <code>#[derive(Diagnostic)]</code>. They are only suitable for simple diagnostics that |
| don't require much logic in deciding whether or not to add additional |
| subdiagnostics.</p> |
| <p>In cases where diagnostics require more complex or dynamic behavior, such as conditionally adding subdiagnostics, |
| customizing the rendering logic, or selecting messages at runtime, you will need to manually implement |
| the corresponding trait (<code>Diagnostic</code>, <code>LintDiagnostic</code>, or <code>Subdiagnostic</code>). |
| This approach provides greater flexibility and is recommended for diagnostics that go beyond simple, static structures.</p> |
| <p>Diagnostic can be translated into different languages and each has a slug that uniquely identifies the diagnostic.</p> |
| <h2 id="derivediagnostic-and-derivelintdiagnostic"><a class="header" href="#derivediagnostic-and-derivelintdiagnostic"><code>#[derive(Diagnostic)]</code> and <code>#[derive(LintDiagnostic)]</code></a></h2> |
| <p>Consider the <a href="https://github.com/rust-lang/rust/blob/6201eabde85db854c1ebb57624be5ec699246b50/compiler/rustc_hir_analysis/src/errors.rs#L68-L77">definition</a> of the "field already declared" diagnostic |
| shown below:</p> |
| <pre><code class="language-rust ignore">#[derive(Diagnostic)] |
| #[diag(hir_analysis_field_already_declared, code = E0124)] |
| pub struct FieldAlreadyDeclared { |
| pub field_name: Ident, |
| #[primary_span] |
| #[label] |
| pub span: Span, |
| #[label(previous_decl_label)] |
| pub prev_span: Span, |
| }</code></pre> |
| <p><code>Diagnostic</code> can only be derived on structs and enums. |
| Attributes that are placed on the type for structs are placed on each |
| variants for enums (or vice versa). Each <code>Diagnostic</code> has to have one |
| attribute, <code>#[diag(...)]</code>, applied to the struct or each enum variant.</p> |
| <p>If an error has an error code (e.g. "E0624"), then that can be specified using |
| the <code>code</code> sub-attribute. Specifying a <code>code</code> isn't mandatory, but if you are |
| porting a diagnostic that uses <code>Diag</code> to use <code>Diagnostic</code> |
| then you should keep the code if there was one.</p> |
| <p><code>#[diag(..)]</code> must provide a slug as the first positional argument (a path to an |
| item in <code>rustc_errors::fluent::*</code>). A slug uniquely identifies the diagnostic |
| and is also how the compiler knows what error message to emit (in the default |
| locale of the compiler, or in the locale requested by the user). See |
| <a href="diagnostics/./translation.html">translation documentation</a> to learn more about how |
| translatable error messages are written and how slug items are generated.</p> |
| <p>In our example, the Fluent message for the "field already declared" diagnostic |
| looks like this:</p> |
| <pre><code class="language-fluent">hir_analysis_field_already_declared = |
| field `{$field_name}` is already declared |
| .label = field already declared |
| .previous_decl_label = `{$field_name}` first declared here |
| </code></pre> |
| <p><code>hir_analysis_field_already_declared</code> is the slug from our example and is followed |
| by the diagnostic message.</p> |
| <p>Every field of the <code>Diagnostic</code> which does not have an annotation is |
| available in Fluent messages as a variable, like <code>field_name</code> in the example |
| above. Fields can be annotated <code>#[skip_arg]</code> if this is undesired.</p> |
| <p>Using the <code>#[primary_span]</code> attribute on a field (that has type <code>Span</code>) |
| indicates the primary span of the diagnostic which will have the main message |
| of the diagnostic.</p> |
| <p>Diagnostics are more than just their primary message, they often include |
| labels, notes, help messages and suggestions, all of which can also be |
| specified on a <code>Diagnostic</code>.</p> |
| <p><code>#[label]</code>, <code>#[help]</code>, <code>#[warning]</code> and <code>#[note]</code> can all be applied to fields which have the |
| type <code>Span</code>. Applying any of these attributes will create the corresponding |
| subdiagnostic with that <code>Span</code>. These attributes will look for their |
| diagnostic message in a Fluent attribute attached to the primary Fluent |
| message. In our example, <code>#[label]</code> will look for |
| <code>hir_analysis_field_already_declared.label</code> (which has the message "field already |
| declared"). If there is more than one subdiagnostic of the same type, then |
| these attributes can also take a value that is the attribute name to look for |
| (e.g. <code>previous_decl_label</code> in our example).</p> |
| <p>Other types have special behavior when used in a <code>Diagnostic</code> derive:</p> |
| <ul> |
| <li>Any attribute applied to an <code>Option<T></code> will only emit a |
| subdiagnostic if the option is <code>Some(..)</code>.</li> |
| <li>Any attribute applied to a <code>Vec<T></code> will be repeated for each element of the |
| vector.</li> |
| </ul> |
| <p><code>#[help]</code>, <code>#[warning]</code> and <code>#[note]</code> can also be applied to the struct itself, in which case |
| they work exactly like when applied to fields except the subdiagnostic won't |
| have a <code>Span</code>. These attributes can also be applied to fields of type <code>()</code> for |
| the same effect, which when combined with the <code>Option</code> type can be used to |
| represent optional <code>#[note]</code>/<code>#[help]</code>/<code>#[warning]</code> subdiagnostics.</p> |
| <p>Suggestions can be emitted using one of four field attributes:</p> |
| <ul> |
| <li><code>#[suggestion(slug, code = "...", applicability = "...")]</code></li> |
| <li><code>#[suggestion_hidden(slug, code = "...", applicability = "...")]</code></li> |
| <li><code>#[suggestion_short(slug, code = "...", applicability = "...")]</code></li> |
| <li><code>#[suggestion_verbose(slug, code = "...", applicability = "...")]</code></li> |
| </ul> |
| <p>Suggestions must be applied on either a <code>Span</code> field or a <code>(Span, MachineApplicability)</code> field. Similarly to other field attributes, the slug |
| specifies the Fluent attribute with the message and defaults to the equivalent |
| of <code>.suggestion</code>. <code>code</code> specifies the code that should be suggested as a |
| replacement and is a format string (e.g. <code>{field_name}</code> would be replaced by |
| the value of the <code>field_name</code> field of the struct), not a Fluent identifier. |
| <code>applicability</code> can be used to specify the applicability in the attribute, it |
| cannot be used when the field's type contains an <code>Applicability</code>.</p> |
| <p>In the end, the <code>Diagnostic</code> derive will generate an implementation of |
| <code>Diagnostic</code> that looks like the following:</p> |
| <pre><code class="language-rust ignore">impl<'a, G: EmissionGuarantee> Diagnostic<'a> for FieldAlreadyDeclared { |
| fn into_diag(self, dcx: &'a DiagCtxt, level: Level) -> Diag<'a, G> { |
| let mut diag = Diag::new(dcx, level, fluent::hir_analysis_field_already_declared); |
| diag.set_span(self.span); |
| diag.span_label( |
| self.span, |
| fluent::hir_analysis_label |
| ); |
| diag.span_label( |
| self.prev_span, |
| fluent::hir_analysis_previous_decl_label |
| ); |
| diag |
| } |
| }</code></pre> |
| <p>Now that we've defined our diagnostic, how do we <a href="https://github.com/rust-lang/rust/blob/f1112099eba41abadb6f921df7edba70affe92c5/compiler/rustc_hir_analysis/src/collect.rs#L823-L827">use it</a>? It's quite |
| straightforward, just create an instance of the struct and pass it to |
| <code>emit_err</code> (or <code>emit_warning</code>):</p> |
| <pre><code class="language-rust ignore">tcx.dcx().emit_err(FieldAlreadyDeclared { |
| field_name: f.ident, |
| span: f.span, |
| prev_span, |
| });</code></pre> |
| <h3 id="reference-for-derivediagnostic-and-derivelintdiagnostic"><a class="header" href="#reference-for-derivediagnostic-and-derivelintdiagnostic">Reference for <code>#[derive(Diagnostic)]</code> and <code>#[derive(LintDiagnostic)]</code></a></h3> |
| <p><code>#[derive(Diagnostic)]</code> and <code>#[derive(LintDiagnostic)]</code> support the |
| following attributes:</p> |
| <ul> |
| <li><code>#[diag(slug, code = "...")]</code> |
| <ul> |
| <li><em>Applied to struct or enum variant.</em></li> |
| <li><em>Mandatory</em></li> |
| <li>Defines the text and error code to be associated with the diagnostic.</li> |
| <li>Slug (<em>Mandatory</em>) |
| <ul> |
| <li>Uniquely identifies the diagnostic and corresponds to its Fluent message, |
| mandatory.</li> |
| <li>A path to an item in <code>rustc_errors::fluent</code>, e.g. |
| <code>rustc_errors::fluent::hir_analysis_field_already_declared</code> |
| (<code>rustc_errors::fluent</code> is implicit in the attribute, so just |
| <code>hir_analysis_field_already_declared</code>).</li> |
| <li>See <a href="diagnostics/./translation.html">translation documentation</a>.</li> |
| </ul> |
| </li> |
| <li><code>code = "..."</code> (<em>Optional</em>) |
| <ul> |
| <li>Specifies the error code.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[note]</code> or <code>#[note(slug)]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to struct or struct fields of type <code>Span</code>, <code>Option<()></code> or <code>()</code>.</em></li> |
| <li>Adds a note subdiagnostic.</li> |
| <li>Value is a path to an item in <code>rustc_errors::fluent</code> for the note's |
| message. |
| <ul> |
| <li>Defaults to equivalent of <code>.note</code>.</li> |
| </ul> |
| </li> |
| <li>If applied to a <code>Span</code> field, creates a spanned note.</li> |
| </ul> |
| </li> |
| <li><code>#[help]</code> or <code>#[help(slug)]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to struct or struct fields of type <code>Span</code>, <code>Option<()></code> or <code>()</code>.</em></li> |
| <li>Adds a help subdiagnostic.</li> |
| <li>Value is a path to an item in <code>rustc_errors::fluent</code> for the note's |
| message. |
| <ul> |
| <li>Defaults to equivalent of <code>.help</code>.</li> |
| </ul> |
| </li> |
| <li>If applied to a <code>Span</code> field, creates a spanned help.</li> |
| </ul> |
| </li> |
| <li><code>#[label]</code> or <code>#[label(slug)]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to <code>Span</code> fields.</em></li> |
| <li>Adds a label subdiagnostic.</li> |
| <li>Value is a path to an item in <code>rustc_errors::fluent</code> for the note's |
| message. |
| <ul> |
| <li>Defaults to equivalent of <code>.label</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[warning]</code> or <code>#[warning(slug)]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to struct or struct fields of type <code>Span</code>, <code>Option<()></code> or <code>()</code>.</em></li> |
| <li>Adds a warning subdiagnostic.</li> |
| <li>Value is a path to an item in <code>rustc_errors::fluent</code> for the note's |
| message. |
| <ul> |
| <li>Defaults to equivalent of <code>.warn</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[suggestion{,_hidden,_short,_verbose}(slug, code = "...", applicability = "...")]</code> |
| (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to <code>(Span, MachineApplicability)</code> or <code>Span</code> fields.</em></li> |
| <li>Adds a suggestion subdiagnostic.</li> |
| <li>Slug (<em>Mandatory</em>) |
| <ul> |
| <li>A path to an item in <code>rustc_errors::fluent</code>, e.g. |
| <code>rustc_errors::fluent::hir_analysis_field_already_declared</code> |
| (<code>rustc_errors::fluent</code> is implicit in the attribute, so just |
| <code>hir_analysis_field_already_declared</code>). Fluent attributes for all messages |
| exist as top-level items in that module (so <code>hir_analysis_message.attr</code> is just |
| <code>attr</code>).</li> |
| <li>See <a href="diagnostics/./translation.html">translation documentation</a>.</li> |
| <li>Defaults to <code>rustc_errors::fluent::_subdiag::suggestion</code> (or</li> |
| <li><code>.suggestion</code> in Fluent).</li> |
| </ul> |
| </li> |
| <li><code>code = "..."</code>/<code>code("...", ...)</code> (<em>Mandatory</em>) |
| <ul> |
| <li>One or multiple format strings indicating the code to be suggested as a |
| replacement. Multiple values signify multiple possible replacements.</li> |
| </ul> |
| </li> |
| <li><code>applicability = "..."</code> (<em>Optional</em>) |
| <ul> |
| <li>String which must be one of <code>machine-applicable</code>, <code>maybe-incorrect</code>, |
| <code>has-placeholders</code> or <code>unspecified</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[subdiagnostic]</code> |
| <ul> |
| <li><em>Applied to a type that implements <code>Subdiagnostic</code> (from |
| <code>#[derive(Subdiagnostic)]</code>).</em></li> |
| <li>Adds the subdiagnostic represented by the subdiagnostic struct.</li> |
| </ul> |
| </li> |
| <li><code>#[primary_span]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to <code>Span</code> fields on <code>Subdiagnostic</code>s. Not used for <code>LintDiagnostic</code>s.</em></li> |
| <li>Indicates the primary span of the diagnostic.</li> |
| </ul> |
| </li> |
| <li><code>#[skip_arg]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to any field.</em></li> |
| <li>Prevents the field from being provided as a diagnostic argument.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="derivesubdiagnostic"><a class="header" href="#derivesubdiagnostic"><code>#[derive(Subdiagnostic)]</code></a></h2> |
| <p>It is common in the compiler to write a function that conditionally adds a |
| specific subdiagnostic to an error if it is applicable. Oftentimes these |
| subdiagnostics could be represented using a diagnostic struct even if the |
| overall diagnostic could not. In this circumstance, the <code>Subdiagnostic</code> |
| derive can be used to represent a partial diagnostic (e.g a note, label, help or |
| suggestion) as a struct.</p> |
| <p>Consider the <a href="https://github.com/rust-lang/rust/blob/f1112099eba41abadb6f921df7edba70affe92c5/compiler/rustc_hir_analysis/src/errors.rs#L221-L234">definition</a> of the "expected return type" label |
| shown below:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>#[derive(Subdiagnostic)] |
| pub enum ExpectedReturnTypeLabel<'tcx> { |
| #[label(hir_analysis_expected_default_return_type)] |
| Unit { |
| #[primary_span] |
| span: Span, |
| }, |
| #[label(hir_analysis_expected_return_type)] |
| Other { |
| #[primary_span] |
| span: Span, |
| expected: Ty<'tcx>, |
| }, |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Like <code>Diagnostic</code>, <code>Subdiagnostic</code> can be derived for structs or |
| enums. Attributes that are placed on the type for structs are placed on each |
| variants for enums (or vice versa). Each <code>Subdiagnostic</code> should have one |
| attribute applied to the struct or each variant, one of:</p> |
| <ul> |
| <li><code>#[label(..)]</code> for defining a label</li> |
| <li><code>#[note(..)]</code> for defining a note</li> |
| <li><code>#[help(..)]</code> for defining a help</li> |
| <li><code>#[warning(..)]</code> for defining a warning</li> |
| <li><code>#[suggestion{,_hidden,_short,_verbose}(..)]</code> for defining a suggestion</li> |
| </ul> |
| <p>All of the above must provide a slug as the first positional argument (a path |
| to an item in <code>rustc_errors::fluent::*</code>). A slug uniquely identifies the |
| diagnostic and is also how the compiler knows what error message to emit (in |
| the default locale of the compiler, or in the locale requested by the user). |
| See <a href="diagnostics/./translation.html">translation documentation</a> to learn more about how |
| translatable error messages are written and how slug items are generated.</p> |
| <p>In our example, the Fluent message for the "expected return type" label |
| looks like this:</p> |
| <pre><code class="language-fluent">hir_analysis_expected_default_return_type = expected `()` because of default return type |
| |
| hir_analysis_expected_return_type = expected `{$expected}` because of return type |
| </code></pre> |
| <p>Using the <code>#[primary_span]</code> attribute on a field (with type <code>Span</code>) will denote |
| the primary span of the subdiagnostic. A primary span is only necessary for a |
| label or suggestion, which can not be spanless.</p> |
| <p>Every field of the type/variant which does not have an annotation is available |
| in Fluent messages as a variable. Fields can be annotated <code>#[skip_arg]</code> if this |
| is undesired.</p> |
| <p>Like <code>Diagnostic</code>, <code>Subdiagnostic</code> supports <code>Option<T></code> and |
| <code>Vec<T></code> fields.</p> |
| <p>Suggestions can be emitted using one of four attributes on the type/variant:</p> |
| <ul> |
| <li><code>#[suggestion(..., code = "...", applicability = "...")]</code></li> |
| <li><code>#[suggestion_hidden(..., code = "...", applicability = "...")]</code></li> |
| <li><code>#[suggestion_short(..., code = "...", applicability = "...")]</code></li> |
| <li><code>#[suggestion_verbose(..., code = "...", applicability = "...")]</code></li> |
| </ul> |
| <p>Suggestions require <code>#[primary_span]</code> be set on a field and can have the |
| following sub-attributes:</p> |
| <ul> |
| <li>The first positional argument specifies the path to a item in |
| <code>rustc_errors::fluent</code> corresponding to the Fluent attribute with the message |
| and defaults to the equivalent of <code>.suggestion</code>.</li> |
| <li><code>code</code> specifies the code that should be suggested as a replacement and is a |
| format string (e.g. <code>{field_name}</code> would be replaced by the value of the |
| <code>field_name</code> field of the struct), not a Fluent identifier.</li> |
| <li><code>applicability</code> can be used to specify the applicability in the attribute, it |
| cannot be used when the field's type contains an <code>Applicability</code>.</li> |
| </ul> |
| <p>Applicabilities can also be specified as a field (of type <code>Applicability</code>) |
| using the <code>#[applicability]</code> attribute.</p> |
| <p>In the end, the <code>Subdiagnostic</code> derive will generate an implementation |
| of <code>Subdiagnostic</code> that looks like the following:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>impl<'tcx> Subdiagnostic for ExpectedReturnTypeLabel<'tcx> { |
| fn add_to_diag(self, diag: &mut rustc_errors::Diagnostic) { |
| use rustc_errors::{Applicability, IntoDiagArg}; |
| match self { |
| ExpectedReturnTypeLabel::Unit { span } => { |
| diag.span_label(span, rustc_errors::fluent::hir_analysis_expected_default_return_type) |
| } |
| ExpectedReturnTypeLabel::Other { span, expected } => { |
| diag.set_arg("expected", expected); |
| diag.span_label(span, rustc_errors::fluent::hir_analysis_expected_return_type) |
| } |
| } |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Once defined, a subdiagnostic can be used by passing it to the <code>subdiagnostic</code> |
| function (<a href="https://github.com/rust-lang/rust/blob/f1112099eba41abadb6f921df7edba70affe92c5/compiler/rustc_hir_analysis/src/check/fn_ctxt/suggestions.rs#L670-L674">example</a> and <a href="https://github.com/rust-lang/rust/blob/f1112099eba41abadb6f921df7edba70affe92c5/compiler/rustc_hir_analysis/src/check/fn_ctxt/suggestions.rs#L704-L707">example</a>) on a |
| diagnostic or by assigning it to a <code>#[subdiagnostic]</code>-annotated field of a |
| diagnostic struct.</p> |
| <h3 id="argument-sharing-and-isolation"><a class="header" href="#argument-sharing-and-isolation">Argument sharing and isolation</a></h3> |
| <p>Subdiagnostics add their own arguments (i.e., certain fields in their structure) to the <code>Diag</code> structure before rendering the information. |
| <code>Diag</code> structure also stores the arguments from the main diagnostic, so the subdiagnostic can also use the arguments from the main diagnostic.</p> |
| <p>However, when a subdiagnostic is added to a main diagnostic by implementing <code>#[derive(Subdiagnostic)]</code>, |
| the following rules, introduced in <a href="https://github.com/rust-lang/rust/pull/142724">rust-lang/rust#142724</a> |
| apply to the handling of arguments (i.e., variables used in Fluent messages):</p> |
| <p><strong>Argument isolation between sub diagnostics</strong>: |
| Arguments set by a subdiagnostic are only available during the rendering of that subdiagnostic. |
| After the subdiagnostic is rendered, all arguments it introduced are restored from the main diagnostic. |
| This ensures that multiple subdiagnostics do not pollute each other's argument scope. |
| For example, when using a <code>Vec<Subdiag></code>, it iteratively adds the same argument over and over again.</p> |
| <p><strong>Same argument override between sub and main diagnostics</strong>: |
| If a subdiagnostic sets a argument with the same name as a arguments already in the main diagnostic, |
| it will report an error at runtime unless both have exactly the same value. |
| It has two benefits:</p> |
| <ul> |
| <li>preserves the flexibility that arguments in the main diagnostic are allowed to appear in the attributes of the subdiagnostic. |
| For example, There is an attribute <code>#[suggestion(code = "{new_vis}")]</code> in the subdiagnostic, but <code>new_vis</code> is the field in the main diagnostic struct.</li> |
| <li>prevents accidental overwriting or deletion of arguments required by the main diagnostic or other subdiagnostics.</li> |
| </ul> |
| <p>These rules guarantee that arguments injected by subdiagnostics are strictly scoped to their own rendering. |
| The main diagnostic's arguments remain unaffected by subdiagnostic logic, even in the presence of name collisions. |
| Additionally, subdiagnostics can access arguments from the main diagnostic with the same name when needed.</p> |
| <h3 id="reference-for-derivesubdiagnostic"><a class="header" href="#reference-for-derivesubdiagnostic">Reference for <code>#[derive(Subdiagnostic)]</code></a></h3> |
| <p><code>#[derive(Subdiagnostic)]</code> supports the following attributes:</p> |
| <ul> |
| <li><code>#[label(slug)]</code>, <code>#[help(slug)]</code>, <code>#[warning(slug)]</code> or <code>#[note(slug)]</code> |
| <ul> |
| <li><em>Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes.</em></li> |
| <li><em>Mandatory</em></li> |
| <li>Defines the type to be representing a label, help or note.</li> |
| <li>Slug (<em>Mandatory</em>) |
| <ul> |
| <li>Uniquely identifies the diagnostic and corresponds to its Fluent message, |
| mandatory.</li> |
| <li>A path to an item in <code>rustc_errors::fluent</code>, e.g. |
| <code>rustc_errors::fluent::hir_analysis_field_already_declared</code> |
| (<code>rustc_errors::fluent</code> is implicit in the attribute, so just |
| <code>hir_analysis_field_already_declared</code>).</li> |
| <li>See <a href="diagnostics/./translation.html">translation documentation</a>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[suggestion{,_hidden,_short,_verbose}(slug, code = "...", applicability = "...")]</code> |
| <ul> |
| <li><em>Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes.</em></li> |
| <li><em>Mandatory</em></li> |
| <li>Defines the type to be representing a suggestion.</li> |
| <li>Slug (<em>Mandatory</em>) |
| <ul> |
| <li>A path to an item in <code>rustc_errors::fluent</code>, e.g. |
| <code>rustc_errors::fluent::hir_analysis_field_already_declared</code> |
| (<code>rustc_errors::fluent</code> is implicit in the attribute, so just |
| <code>hir_analysis::field_already_declared</code>). Fluent attributes for all messages |
| exist as top-level items in that module (so <code>hir_analysis_message.attr</code> is just |
| <code>hir_analysis::attr</code>).</li> |
| <li>See <a href="diagnostics/./translation.html">translation documentation</a>.</li> |
| <li>Defaults to <code>rustc_errors::fluent::_subdiag::suggestion</code> (or</li> |
| <li><code>.suggestion</code> in Fluent).</li> |
| </ul> |
| </li> |
| <li><code>code = "..."</code>/<code>code("...", ...)</code> (<em>Mandatory</em>) |
| <ul> |
| <li>One or multiple format strings indicating the code to be suggested as a |
| replacement. Multiple values signify multiple possible replacements.</li> |
| </ul> |
| </li> |
| <li><code>applicability = "..."</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Mutually exclusive with <code>#[applicability]</code> on a field.</em></li> |
| <li>Value is the applicability of the suggestion.</li> |
| <li>String which must be one of: |
| <ul> |
| <li><code>machine-applicable</code></li> |
| <li><code>maybe-incorrect</code></li> |
| <li><code>has-placeholders</code></li> |
| <li><code>unspecified</code></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[multipart_suggestion{,_hidden,_short,_verbose}(slug, applicability = "...")]</code> |
| <ul> |
| <li><em>Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes.</em></li> |
| <li><em>Mandatory</em></li> |
| <li>Defines the type to be representing a multipart suggestion.</li> |
| <li>Slug (<em>Mandatory</em>): see <code>#[suggestion]</code></li> |
| <li><code>applicability = "..."</code> (<em>Optional</em>): see <code>#[suggestion]</code></li> |
| </ul> |
| </li> |
| <li><code>#[primary_span]</code> (<em>Mandatory</em> for labels and suggestions; <em>optional</em> otherwise; not applicable |
| to multipart suggestions) |
| <ul> |
| <li><em>Applied to <code>Span</code> fields.</em></li> |
| <li>Indicates the primary span of the subdiagnostic.</li> |
| </ul> |
| </li> |
| <li><code>#[suggestion_part(code = "...")]</code> (<em>Mandatory</em>; only applicable to multipart suggestions) |
| <ul> |
| <li><em>Applied to <code>Span</code> fields.</em></li> |
| <li>Indicates the span to be one part of the multipart suggestion.</li> |
| <li><code>code = "..."</code> (<em>Mandatory</em>) |
| <ul> |
| <li>Value is a format string indicating the code to be suggested as a |
| replacement.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><code>#[applicability]</code> (<em>Optional</em>; only applicable to (simple and multipart) suggestions) |
| <ul> |
| <li><em>Applied to <code>Applicability</code> fields.</em></li> |
| <li>Indicates the applicability of the suggestion.</li> |
| </ul> |
| </li> |
| <li><code>#[skip_arg]</code> (<em>Optional</em>) |
| <ul> |
| <li><em>Applied to any field.</em></li> |
| <li>Prevents the field from being provided as a diagnostic argument.</li> |
| </ul> |
| </li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="translation"><a class="header" href="#translation">Translation</a></h1> |
| <div class="warning"> |
| rustc's current diagnostics translation infrastructure (as of |
| <!-- date-check --> October 2024 |
| ) unfortunately causes some friction for compiler contributors, and the current |
| infrastructure is mostly pending a redesign that better addresses needs of both |
| compiler contributors and translation teams. Note that there is no current |
| active redesign proposals (as of |
| <!-- date-check --> October 2024 |
| )! |
| <p>Please see the tracking issue <a href="https://github.com/rust-lang/rust/issues/132181">https://github.com/rust-lang/rust/issues/132181</a> |
| for status updates.</p> |
| <p>We have downgraded the internal lints <code>untranslatable_diagnostic</code> and |
| <code>diagnostic_outside_of_impl</code>. Those internal lints previously required new code |
| to use the current translation infrastructure. However, because the translation |
| infra is waiting for a yet-to-be-proposed redesign and thus rework, we are not |
| mandating usage of current translation infra. Use the infra if you <em>want to</em> or |
| otherwise makes the code cleaner, but otherwise sidestep the translation infra |
| if you need more flexibility.</p> |
| </div> |
| <p>rustc's diagnostic infrastructure supports translatable diagnostics using |
| <a href="https://projectfluent.org">Fluent</a>.</p> |
| <h2 id="writing-translatable-diagnostics"><a class="header" href="#writing-translatable-diagnostics">Writing translatable diagnostics</a></h2> |
| <p>There are two ways of writing translatable diagnostics:</p> |
| <ol> |
| <li>For simple diagnostics, using a diagnostic (or subdiagnostic) derive. |
| ("Simple" diagnostics being those that don't require a lot of logic in |
| deciding to emit subdiagnostics and can therefore be represented as |
| diagnostic structs). See <a href="diagnostics/./diagnostic-structs.html">the diagnostic and subdiagnostic structs |
| documentation</a>.</li> |
| <li>Using typed identifiers with <code>Diag</code> APIs (in |
| <code>Diagnostic</code> or <code>Subdiagnostic</code> or <code>LintDiagnostic</code> implementations).</li> |
| </ol> |
| <p>When adding or changing a translatable diagnostic, |
| you don't need to worry about the translations. |
| Only updating the original English message is required. |
| Currently, |
| each crate which defines translatable diagnostics has its own Fluent resource, |
| which is a file named <code>messages.ftl</code>, |
| located in the root of the crate |
| (such as<code>compiler/rustc_expand/messages.ftl</code>).</p> |
| <h2 id="fluent"><a class="header" href="#fluent">Fluent</a></h2> |
| <p>Fluent is built around the idea of "asymmetric localization", which aims to |
| decouple the expressiveness of translations from the grammar of the source |
| language (English in rustc's case). Prior to translation, rustc's diagnostics |
| relied heavily on interpolation to build the messages shown to the users. |
| Interpolated strings are hard to translate because writing a natural-sounding |
| translation might require more, less, or just different interpolation than the |
| English string, all of which would require changes to the compiler's source |
| code to support.</p> |
| <p>Diagnostic messages are defined in Fluent resources. A combined set of Fluent |
| resources for a given locale (e.g. <code>en-US</code>) is known as Fluent bundle.</p> |
| <pre><code class="language-fluent">typeck_address_of_temporary_taken = cannot take address of a temporary |
| </code></pre> |
| <p>In the above example, <code>typeck_address_of_temporary_taken</code> is the identifier for |
| a Fluent message and corresponds to the diagnostic message in English. Other |
| Fluent resources can be written which would correspond to a message in another |
| language. Each diagnostic therefore has at least one Fluent message.</p> |
| <pre><code class="language-fluent">typeck_address_of_temporary_taken = cannot take address of a temporary |
| .label = temporary value |
| </code></pre> |
| <p>By convention, diagnostic messages for subdiagnostics are specified as |
| "attributes" on Fluent messages (additional related messages, denoted by the |
| <code>.<attribute-name></code> syntax). In the above example, <code>label</code> is an attribute of |
| <code>typeck_address_of_temporary_taken</code> which corresponds to the message for the |
| label added to this diagnostic.</p> |
| <p>Diagnostic messages often interpolate additional context into the message shown |
| to the user, such as the name of a type or of a variable. Additional context to |
| Fluent messages is provided as an "argument" to the diagnostic.</p> |
| <pre><code class="language-fluent">typeck_struct_expr_non_exhaustive = |
| cannot create non-exhaustive {$what} using struct expression |
| </code></pre> |
| <p>In the above example, the Fluent message refers to an argument named <code>what</code> |
| which is expected to exist (how arguments are provided to diagnostics is |
| discussed in detail later).</p> |
| <p>You can consult the <a href="https://projectfluent.org">Fluent</a> documentation for other usage examples of Fluent |
| and its syntax.</p> |
| <h3 id="guideline-for-message-naming"><a class="header" href="#guideline-for-message-naming">Guideline for message naming</a></h3> |
| <p>Usually, fluent uses <code>-</code> for separating words inside a message name. However, |
| <code>_</code> is accepted by fluent as well. As <code>_</code> fits Rust's use cases better, due to |
| the identifiers on the Rust side using <code>_</code> as well, inside rustc, <code>-</code> is not |
| allowed for separating words, and instead <code>_</code> is recommended. The only exception |
| is for leading <code>-</code>s, for message names like <code>-passes_see_issue</code>.</p> |
| <h3 id="guidelines-for-writing-translatable-messages"><a class="header" href="#guidelines-for-writing-translatable-messages">Guidelines for writing translatable messages</a></h3> |
| <p>For a message to be translatable into different languages, all of the |
| information required by any language must be provided to the diagnostic as an |
| argument (not just the information required in the English message).</p> |
| <p>As the compiler team gain more experience writing diagnostics that have all of |
| the information necessary to be translated into different languages, this page |
| will be updated with more guidance. For now, the <a href="https://projectfluent.org">Fluent</a> documentation has |
| excellent examples of translating messages into different locales and the |
| information that needs to be provided by the code to do so.</p> |
| <h3 id="compile-time-validation-and-typed-identifiers"><a class="header" href="#compile-time-validation-and-typed-identifiers">Compile-time validation and typed identifiers</a></h3> |
| <p>rustc's <code>fluent_messages</code> macro performs compile-time validation of Fluent |
| resources and generates code to make it easier to refer to Fluent messages in |
| diagnostics.</p> |
| <p>Compile-time validation of Fluent resources will emit any parsing errors |
| from Fluent resources while building the compiler, preventing invalid Fluent |
| resources from causing panics in the compiler. Compile-time validation also |
| emits an error if multiple Fluent messages have the same identifier.</p> |
| <h2 id="internals"><a class="header" href="#internals">Internals</a></h2> |
| <p>Various parts of rustc's diagnostic internals are modified in order to support |
| translation.</p> |
| <h3 id="messages"><a class="header" href="#messages">Messages</a></h3> |
| <p>All of rustc's traditional diagnostic APIs (e.g. <code>struct_span_err</code> or <code>note</code>) |
| take any message that can be converted into a <code>DiagMessage</code> (or |
| <code>SubdiagMessage</code>).</p> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_error_messages/enum.DiagMessage.html"><code>rustc_error_messages::DiagMessage</code></a> can represent legacy non-translatable |
| diagnostic messages and translatable messages. Non-translatable messages are |
| just <code>String</code>s. Translatable messages are just a <code>&'static str</code> with the |
| identifier of the Fluent message (sometimes with an additional <code>&'static str</code> |
| with an attribute).</p> |
| <p><code>DiagMessage</code> never needs to be interacted with directly: |
| <code>DiagMessage</code> constants are created for each diagnostic message in a |
| Fluent resource (described in more detail below), or <code>DiagMessage</code>s will |
| either be created in the macro-generated code of a diagnostic derive.</p> |
| <p><code>rustc_error_messages::SubdiagMessage</code> is similar, it can correspond to a |
| legacy non-translatable diagnostic message or the name of an attribute to a |
| Fluent message. Translatable <code>SubdiagMessage</code>s must be combined with a |
| <code>DiagMessage</code> (using <code>DiagMessage::with_subdiagnostic_message</code>) to |
| be emitted (an attribute name on its own is meaningless without a corresponding |
| message identifier, which is what <code>DiagMessage</code> provides).</p> |
| <p>Both <code>DiagMessage</code> and <code>SubdiagMessage</code> implement <code>Into</code> for any |
| type that can be converted into a string, and converts these into |
| non-translatable diagnostics - this keeps all existing diagnostic calls |
| working.</p> |
| <h3 id="arguments"><a class="header" href="#arguments">Arguments</a></h3> |
| <p>Additional context for Fluent messages which are interpolated into message |
| contents needs to be provided to translatable diagnostics.</p> |
| <p>Diagnostics have a <code>set_arg</code> function that can be used to provide this |
| additional context to a diagnostic.</p> |
| <p>Arguments have both a name (e.g. "what" in the earlier example) and a value. |
| Argument values are represented using the <code>DiagArgValue</code> type, which is |
| just a string or a number. rustc types can implement <code>IntoDiagArg</code> with |
| conversion into a string or a number, and common types like <code>Ty<'tcx></code> already |
| have such implementations.</p> |
| <p><code>set_arg</code> calls are handled transparently by diagnostic derives but need to be |
| added manually when using diagnostic builder APIs.</p> |
| <h3 id="loading"><a class="header" href="#loading">Loading</a></h3> |
| <p>rustc makes a distinction between the "fallback bundle" for <code>en-US</code> that is used |
| by default and when another locale is missing a message; and the primary fluent |
| bundle which is requested by the user.</p> |
| <p>Diagnostic emitters implement the <code>Emitter</code> trait which has two functions for |
| accessing the fallback and primary fluent bundles (<code>fallback_fluent_bundle</code> and |
| <code>fluent_bundle</code> respectively).</p> |
| <p><code>Emitter</code> also has member functions with default implementations for performing |
| translation of a <code>DiagMessage</code> using the results of |
| <code>fallback_fluent_bundle</code> and <code>fluent_bundle</code>.</p> |
| <p>All of the emitters in rustc load the fallback Fluent bundle lazily, only |
| reading Fluent resources and parsing them when an error message is first being |
| translated (for performance reasons - it doesn't make sense to do this if no |
| error is being emitted). <code>rustc_error_messages::fallback_fluent_bundle</code> returns |
| a <code>std::lazy::Lazy<FluentBundle></code> which is provided to emitters and evaluated |
| in the first call to <code>Emitter::fallback_fluent_bundle</code>.</p> |
| <p>The primary Fluent bundle (for the user's desired locale) is expected to be |
| returned by <code>Emitter::fluent_bundle</code>. This bundle is used preferentially when |
| translating messages, the fallback bundle is only used if the primary bundle is |
| missing a message or not provided.</p> |
| <p>There are no locale bundles distributed with the compiler, |
| but mechanisms are implemented for loading them.</p> |
| <ul> |
| <li><code>-Ztranslate-additional-ftl</code> can be used to load a specific resource as the |
| primary bundle for testing purposes.</li> |
| <li><code>-Ztranslate-lang</code> can be provided a language identifier (something like |
| <code>en-US</code>) and will load any Fluent resources found in |
| <code>$sysroot/share/locale/$locale/</code> directory (both the user provided |
| sysroot and any sysroot candidates).</li> |
| </ul> |
| <p>Primary bundles are not currently loaded lazily and if requested will be loaded |
| at the start of compilation regardless of whether an error occurs. Lazily |
| loading primary bundles is possible if it can be assumed that loading a bundle |
| won't fail. Bundle loading can fail if a requested locale is missing, Fluent |
| files are malformed, or a message is duplicated in multiple resources.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="lints-2"><a class="header" href="#lints-2">Lints</a></h1> |
| <p>This page documents some of the machinery around lint registration and how we |
| run lints in the compiler.</p> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LintStore.html"><code>LintStore</code></a> is the central piece of infrastructure, around which |
| everything rotates. The <code>LintStore</code> is held as part of the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html"><code>Session</code></a>, and it |
| gets populated with the list of lints shortly after the <code>Session</code> is created.</p> |
| <h2 id="lints-vs-lint-passes"><a class="header" href="#lints-vs-lint-passes">Lints vs. lint passes</a></h2> |
| <p>There are two parts to the linting mechanism within the compiler: lints and |
| lint passes. Unfortunately, a lot of the documentation we have refers to both |
| of these as just "lints."</p> |
| <p>First, we have the lint declarations themselves, |
| and this is where the name and default lint level and other metadata come from. |
| These are normally defined by way of the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/macro.declare_lint.html"><code>declare_lint!</code></a> macro, |
| which boils down to a static with type <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/struct.Lint.html"><code>&rustc_lint_defs::Lint</code></a> |
| (although this may change in the future, |
| as the macro is somewhat unwieldy to add new fields to, |
| like all macros).</p> |
| <p>As of <!-- date-check --> Aug 2022, |
| we lint against direct declarations without the use of the macro.</p> |
| <p>Lint declarations don't carry any "state" - they are merely global identifiers |
| and descriptions of lints. We assert at runtime that they are not registered |
| twice (by lint name).</p> |
| <p>Lint passes are the meat of any lint. Notably, there is not a one-to-one |
| relationship between lints and lint passes; a lint might not have any lint pass |
| that emits it, it could have many, or just one -- the compiler doesn't track |
| whether a pass is in any way associated with a particular lint, and frequently |
| lints are emitted as part of other work (e.g., type checking, etc.).</p> |
| <h2 id="registration"><a class="header" href="#registration">Registration</a></h2> |
| <h3 id="high-level-overview"><a class="header" href="#high-level-overview">High-level overview</a></h3> |
| <p>In <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html#reexport.run_compiler"><code>rustc_interface::run_compiler</code></a>, |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LintStore.html"><code>LintStore</code></a> is created, |
| and all lints are registered.</p> |
| <p>There are three 'sources' of lints:</p> |
| <ul> |
| <li>internal lints: lints only used by the rustc codebase</li> |
| <li>builtin lints: lints built into the compiler and not provided by some outside |
| source</li> |
| <li><code>rustc_interface::Config</code><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Config.html#structfield.register_lints"><code>register_lints</code></a>: lints passed into the compiler |
| during construction</li> |
| </ul> |
| <p>Lints are registered via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LintStore.html#method.register_lints"><code>LintStore::register_lint</code></a> function. This should |
| happen just once for any lint, or an ICE will occur.</p> |
| <p>Once the registration is complete, we "freeze" the lint store by placing it in |
| an <code>Arc</code>.</p> |
| <p>Lint passes are registered separately into one of the categories |
| (pre-expansion, early, late, late module). Passes are registered as a closure |
| -- i.e., <code>impl Fn() -> Box<dyn X></code>, where <code>dyn X</code> is either an early or late |
| lint pass trait object. When we run the lint passes, we run the closure and |
| then invoke the lint pass methods. The lint pass methods take <code>&mut self</code> so |
| they can keep track of state internally.</p> |
| <h4 id="internal-lints"><a class="header" href="#internal-lints">Internal lints</a></h4> |
| <p>These are lints used just by the compiler or drivers like <code>clippy</code>. They can be |
| found in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/internal/index.html"><code>rustc_lint::internal</code></a>.</p> |
| <p>An example of such a lint is the check that lint passes are implemented using |
| the <code>declare_lint_pass!</code> macro and not by hand. This is accomplished with the |
| <code>LINT_PASS_IMPL_WITHOUT_MACRO</code> lint.</p> |
| <p>Registration of these lints happens in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.register_internals.html"><code>rustc_lint::register_internals</code></a> |
| function which is called when constructing a new lint store inside |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.new_lint_store.html"><code>rustc_lint::new_lint_store</code></a>.</p> |
| <h4 id="builtin-lints"><a class="header" href="#builtin-lints">Builtin Lints</a></h4> |
| <p>These are primarily described in two places, |
| <code>rustc_lint_defs::builtin</code> and <code>rustc_lint::builtin</code>. |
| Often the first provides the definitions for the lints themselves, |
| and the latter provides the lint pass definitions (and implementations), |
| but this is not always true.</p> |
| <p>The builtin lint registration happens in |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.register_builtins.html"><code>rustc_lint::register_builtins</code></a> function. |
| Just like with internal lints, |
| this happens inside of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.new_lint_store.html"><code>rustc_lint::new_lint_store</code></a>.</p> |
| <h4 id="driver-lints"><a class="header" href="#driver-lints">Driver lints</a></h4> |
| <p>These are the lints provided by drivers via the <code>rustc_interface::Config</code> |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Config.html#structfield.register_lints"><code>register_lints</code></a> field, which is a callback. Drivers should, if finding it |
| already set, call the function currently set within the callback they add. The |
| best way for drivers to get access to this is by overriding the |
| <code>Callbacks::config</code> function which gives them direct access to the <code>Config</code> |
| structure.</p> |
| <h2 id="compiler-lint-passes-are-combined-into-one-pass"><a class="header" href="#compiler-lint-passes-are-combined-into-one-pass">Compiler lint passes are combined into one pass</a></h2> |
| <p>Within the compiler, for performance reasons, we usually do not register dozens |
| of lint passes. Instead, we have a single lint pass of each variety (e.g., |
| <code>BuiltinCombinedModuleLateLintPass</code>) which will internally call all of the |
| individual lint passes; this is because then we get the benefits of static over |
| dynamic dispatch for each of the (often empty) trait methods.</p> |
| <p>Ideally, we'd not have to do this, since it adds to the complexity of |
| understanding the code. However, with the current type-erased lint store |
| approach, it is beneficial to do so for performance reasons.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="error-codes"><a class="header" href="#error-codes">Error codes</a></h1> |
| <p>We generally try to assign each error message a unique code like <code>E0123</code>. These |
| codes are defined in the compiler in the <code>diagnostics.rs</code> files found in each |
| crate, which basically consist of macros. All error codes have an associated |
| explanation: new error codes must include them. Note that not all <em>historical</em> |
| (no longer emitted) error codes have explanations.</p> |
| <h2 id="error-explanations"><a class="header" href="#error-explanations">Error explanations</a></h2> |
| <p>The explanations are written in Markdown (see the <a href="https://spec.commonmark.org/current/">CommonMark Spec</a> for |
| specifics around syntax), and all of them are linked in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_error_codes/index.html"><code>rustc_error_codes</code></a> |
| crate. Please read <a href="https://github.com/rust-lang/rfcs/blob/master/text/1567-long-error-codes-explanation-normalization.md">RFC 1567</a> for details on how to format and write long error |
| codes. As of <!-- date-check --> February 2023, there is an |
| effort<sup class="footnote-reference" id="fr-new-explanations-1"><a href="#footnote-new-explanations">1</a></sup> to replace this largely outdated RFC with a new more |
| flexible standard.</p> |
| <p>Error explanations should expand on the error message and provide details about |
| <em>why</em> the error occurs. It is not helpful for users to copy-paste a quick fix; |
| explanations should help users understand why their code cannot be accepted by |
| the compiler. Rust prides itself on helpful error messages and long-form |
| explanations are no exception. However, before error explanations are |
| overhauled<sup class="footnote-reference" id="fr-new-explanations-2"><a href="#footnote-new-explanations">1</a></sup> it is a bit open as to how exactly they should be |
| written, as always: ask your reviewer or ask around on the Rust Zulip.</p> |
| <h2 id="allocating-a-fresh-code"><a class="header" href="#allocating-a-fresh-code">Allocating a fresh code</a></h2> |
| <p>Error codes are stored in <code>compiler/rustc_error_codes</code>.</p> |
| <p>To create a new error, you first need to find the next available |
| code. You can find it with <code>tidy</code>:</p> |
| <pre><code>./x test tidy |
| </code></pre> |
| <p>This will invoke the tidy script, which generally checks that your code obeys |
| our coding conventions. Some of these jobs check error codes and ensure that |
| there aren't duplicates, etc (the tidy check is defined in |
| <code>src/tools/tidy/src/error_codes.rs</code>). Once it is finished with that, tidy will |
| print out the highest used error code:</p> |
| <pre><code>... |
| tidy check |
| Found 505 error codes |
| Highest error code: `E0591` |
| ... |
| </code></pre> |
| <p>Here we see the highest error code in use is <code>E0591</code>, so we <em>probably</em> want |
| <code>E0592</code>. To be sure, run <code>rg E0592</code> and check, you should see no references.</p> |
| <p>You will have to write an extended description for your error, |
| which will go in <code>rustc_error_codes/src/error_codes/E0592.md</code>. |
| To register the error, open <code>rustc_error_codes/src/error_codes.rs</code> and add the |
| code (in its proper numerical order) into<code> register_diagnostics!</code> macro, like |
| this:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>register_diagnostics! { |
| ... |
| E0592: include_str!("./error_codes/E0592.md"), |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>To actually issue the error, you can use the <code>struct_span_code_err!</code> macro:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct_span_code_err!(self.dcx(), // some path to the `DiagCtxt` here |
| span, // whatever span in the source you want |
| E0592, // your new error code |
| fluent::example::an_error_message) |
| .emit() // actually issue the error |
| <span class="boring">}</span></code></pre></pre> |
| <p>If you want to add notes or other snippets, you can invoke methods before you |
| call <code>.emit()</code>:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct_span_code_err!(...) |
| .span_label(another_span, fluent::example::example_label) |
| .span_note(another_span, fluent::example::separate_note) |
| .emit() |
| <span class="boring">}</span></code></pre></pre> |
| <p>For an example of a PR adding an error code, see <a href="https://github.com/rust-lang/rust/pull/76143">#76143</a>.</p> |
| <h2 id="running-error-code-doctests"><a class="header" href="#running-error-code-doctests">Running error code doctests</a></h2> |
| <p>To test the examples added in <code>rustc_error_codes/src/error_codes</code>, run the |
| error index generator using:</p> |
| <pre><code>./x test ./src/tools/error_index_generator |
| </code></pre> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-new-explanations"> |
| <p>See the draft RFC <a href="https://github.com/rust-lang/rfcs/pull/3370">here</a>. <a href="#fr-new-explanations-1">↩</a> <a href="#fr-new-explanations-2">↩2</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="diagnostic-items"><a class="header" href="#diagnostic-items">Diagnostic Items</a></h1> |
| <p>While writing lints it's common to check for specific types, traits and |
| functions. This raises the question on how to check for these. Types can be |
| checked by their complete type path. However, this requires hard coding paths |
| and can lead to misclassifications in some edge cases. To counteract this, |
| rustc has introduced diagnostic items that are used to identify types via |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html"><code>Symbol</code></a>s.</p> |
| <h2 id="finding-diagnostic-items"><a class="header" href="#finding-diagnostic-items">Finding diagnostic items</a></h2> |
| <p>Diagnostic items are added to items inside <code>rustc</code>/<code>std</code>/<code>core</code>/<code>alloc</code> with the |
| <code>rustc_diagnostic_item</code> attribute. The item for a specific type can be found by |
| opening the source code in the documentation and looking for this attribute. |
| Note that it's often added with the <code>cfg_attr</code> attribute to avoid compilation |
| errors during tests. A definition often looks like this:</p> |
| <pre><code class="language-rs">// This is the diagnostic item for this type vvvvvvv |
| #[cfg_attr(not(test), rustc_diagnostic_item = "Penguin")] |
| struct Penguin; |
| </code></pre> |
| <p>Diagnostic items are usually only added to traits, |
| types, |
| and standalone functions. |
| If the goal is to check for an associated type or method, |
| please use the diagnostic item of the item and reference |
| <a href="diagnostics/diagnostic-items.html#using-diagnostic-items"><em>Using Diagnostic Items</em></a>.</p> |
| <h2 id="adding-diagnostic-items"><a class="header" href="#adding-diagnostic-items">Adding diagnostic items</a></h2> |
| <p>A new diagnostic item can be added with these two steps:</p> |
| <ol> |
| <li> |
| <p>Find the target item inside the Rust repo. Now add the diagnostic item as a |
| string via the <code>rustc_diagnostic_item</code> attribute. This can sometimes cause |
| compilation errors while running tests. These errors can be avoided by using |
| the <code>cfg_attr</code> attribute with the <code>not(test)</code> condition (it's fine adding |
| then for all <code>rustc_diagnostic_item</code> attributes as a preventive manner). At |
| the end, it should look like this:</p> |
| <pre><code class="language-rs">// This will be the new diagnostic item vvv |
| #[cfg_attr(not(test), rustc_diagnostic_item = "Cat")] |
| struct Cat; |
| </code></pre> |
| <p>For the naming conventions of diagnostic items, please refer to |
| <a href="diagnostics/diagnostic-items.html#naming-conventions"><em>Naming Conventions</em></a>.</p> |
| </li> |
| <li><!-- date-check: Feb 2023 --> |
| <p>Diagnostic items in code are accessed via symbols in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/sym/index.html"><code>rustc_span::symbol::sym</code></a>. |
| To add your newly-created diagnostic item, |
| simply open the module file, |
| and add the name (In this case <code>Cat</code>) at the correct point in the list.</p> |
| </li> |
| </ol> |
| <p>Now you can create a pull request with your changes. :tada:</p> |
| <blockquote> |
| <p>NOTE: |
| When using diagnostic items in other projects like Clippy, |
| it might take some time until the repos get synchronized.</p> |
| </blockquote> |
| <h2 id="naming-conventions-1"><a class="header" href="#naming-conventions-1">Naming conventions</a></h2> |
| <p>Diagnostic items don't have a naming convention yet. |
| Following are some guidelines that should be used in future, |
| but might differ from existing names:</p> |
| <ul> |
| <li>Types, traits, and enums are named using UpperCamelCase |
| (Examples: <code>Iterator</code> and <code>HashMap</code>)</li> |
| <li>For type names that are used multiple times, |
| like <code>Writer</code>, |
| it's good to choose a more precise name, |
| maybe by adding the module to it |
| (Example: <code>IoWriter</code>)</li> |
| <li>Associated items should not get their own diagnostic items, |
| but instead be accessed indirectly by the diagnostic item |
| of the type they're originating from.</li> |
| <li>Freestanding functions like <code>std::mem::swap()</code> should be named using |
| <code>snake_case</code> with one important (export) module as a prefix |
| (Examples: <code>mem_swap</code> and <code>cmp_max</code>)</li> |
| <li>Modules should usually not have a diagnostic item attached to them. |
| Diagnostic items were added to avoid the usage of paths, |
| and using them on modules would therefore most likely be counterproductive.</li> |
| </ul> |
| <h2 id="using-diagnostic-items"><a class="header" href="#using-diagnostic-items">Using diagnostic items</a></h2> |
| <p>In rustc, diagnostic items are looked up via <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/struct.Symbol.html"><code>Symbol</code></a>s from inside the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/symbol/sym/index.html"><code>rustc_span::symbol::sym</code></a> module. These can then be mapped to <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a>s |
| using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.get_diagnostic_item"><code>TyCtxt::get_diagnostic_item()</code></a> or checked if they match a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a> |
| using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.is_diagnostic_item"><code>TyCtxt::is_diagnostic_item()</code></a>. When mapping from a diagnostic item to |
| a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a>, the method will return a <code>Option<DefId></code>. This can be <code>None</code> if |
| either the symbol isn't a diagnostic item or the type is not registered, for |
| instance when compiling with <code>#[no_std]</code>. |
| All the following examples are based on <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a>s and their usage.</p> |
| <h3 id="example-checking-for-a-type"><a class="header" href="#example-checking-for-a-type">Example: Checking for a type</a></h3> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>use rustc_span::symbol::sym; |
| |
| /// This example checks if the given type (`ty`) has the type `HashMap` using |
| /// `TyCtxt::is_diagnostic_item()` |
| fn example_1(cx: &LateContext<'_>, ty: Ty<'_>) -> bool { |
| match ty.kind() { |
| ty::Adt(adt, _) => cx.tcx.is_diagnostic_item(sym::HashMap, adt.did()), |
| _ => false, |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="example-checking-for-a-trait-implementation"><a class="header" href="#example-checking-for-a-trait-implementation">Example: Checking for a trait implementation</a></h3> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>/// This example checks if a given [`DefId`] from a method is part of a trait |
| /// implementation defined by a diagnostic item. |
| fn is_diag_trait_item( |
| cx: &LateContext<'_>, |
| def_id: DefId, |
| diag_item: Symbol |
| ) -> bool { |
| if let Some(trait_did) = cx.tcx.trait_of_item(def_id) { |
| return cx.tcx.is_diagnostic_item(diag_item, trait_did); |
| } |
| false |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="associated-types"><a class="header" href="#associated-types">Associated Types</a></h3> |
| <p>Associated types of diagnostic items can be accessed indirectly by first |
| getting the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html"><code>DefId</code></a> of the trait and then calling |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.associated_items"><code>TyCtxt::associated_items()</code></a>. This returns an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/assoc/struct.AssocItems.html"><code>AssocItems</code></a> object which can |
| be used for further checks. Checkout |
| <a href="https://github.com/rust-lang/rust-clippy/blob/305177342fbc622c0b3cb148467bab4b9524c934/clippy_utils/src/ty.rs#L55-L72"><code>clippy_utils::ty::get_iterator_item_ty()</code></a> for an example usage of this.</p> |
| <h3 id="usage-in-clippy"><a class="header" href="#usage-in-clippy">Usage in Clippy</a></h3> |
| <p>Clippy tries to use diagnostic items where possible and has developed some |
| wrapper and utility functions. Please also refer to its documentation when |
| using diagnostic items in Clippy. (See <a href="https://doc.rust-lang.org/nightly/clippy/development/common_tools_writing_lints.html"><em>Common tools for writing |
| lints</em></a>.)</p> |
| <h2 id="related-issues"><a class="header" href="#related-issues">Related issues</a></h2> |
| <p>These are probably only interesting to people |
| who really want to take a deep dive into the topic :)</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/pull/60966">rust#60966</a>: The Rust PR that introduced diagnostic items</li> |
| <li><a href="https://github.com/rust-lang/rust-clippy/issues/5393">rust-clippy#5393</a>: Clippy's tracking issue for moving away from hard coded paths to |
| diagnostic item</li> |
| </ul> |
| <!-- Links --> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="errorguaranteed"><a class="header" href="#errorguaranteed"><code>ErrorGuaranteed</code></a></h1> |
| <p>The previous sections have been about the error message that a user of the |
| compiler sees. But emitting an error can also have a second important side |
| effect within the compiler source code: it generates an |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.ErrorGuaranteed.html"><code>ErrorGuaranteed</code></a>.</p> |
| <p><code>ErrorGuaranteed</code> is a zero-sized type that is unconstructable outside of the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/index.html"><code>rustc_errors</code></a> crate. It is generated whenever an error is reported |
| to the user, so that if your compiler code ever encounters a value of type |
| <code>ErrorGuaranteed</code>, the compilation is <em>statically guaranteed to fail</em>. This is |
| useful for avoiding unsoundness bugs because you can statically check that an |
| error code path leads to a failure.</p> |
| <p>There are some important considerations about the usage of <code>ErrorGuaranteed</code>:</p> |
| <ul> |
| <li>It does <em>not</em> convey information about the <em>kind</em> of error. For example, the |
| error may be due (indirectly) to a delayed bug or other compiler error. |
| Thus, you should not rely on |
| <code>ErrorGuaranteed</code> when deciding whether to emit an error, or what kind of error |
| to emit.</li> |
| <li><code>ErrorGuaranteed</code> should not be used to indicate that a compilation <em>will |
| emit</em> an error in the future. It should be used to indicate that an error |
| <em>has already been</em> emitted -- that is, the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/diagnostic/struct.Diag.html#method.emit"><code>emit()</code></a> function has |
| already been called. For example, if we detect that a future part of the |
| compiler will error, we <em>cannot</em> use <code>ErrorGuaranteed</code> unless we first emit |
| an error or delayed bug ourselves.</li> |
| </ul> |
| <p>Thankfully, in most cases, it should be statically impossible to abuse |
| <code>ErrorGuaranteed</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="analysis-1"><a class="header" href="#analysis-1">Analysis</a></h1> |
| <p>This part discusses the many analyses that the compiler uses to check various |
| properties of the code and to inform later stages. Typically, this is what people |
| mean when they talk about "Rust's type system". This includes the |
| representation, inference, and checking of types, the trait system, and the |
| borrow checker. These analyses do not happen as one big pass or set of |
| contiguous passes. Rather, they are spread out throughout various parts of the |
| compilation process and use different intermediate representations. For example, |
| type checking happens on the HIR, while borrow checking happens on the MIR. |
| Nonetheless, for the sake of presentation, we will discuss all of these |
| analyses in this part of the guide.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="generic-parameter-definitions"><a class="header" href="#generic-parameter-definitions">Generic parameter definitions</a></h1> |
| <p>This chapter will discuss how rustc tracks what generic parameters are introduced. For example given some <code>struct Foo<T></code> how does rustc track that <code>Foo</code> defines some type parameter <code>T</code> (and no other generic parameters).</p> |
| <p>This will <em>not</em> cover how we track generic parameters introduced via <code>for<'a></code> syntax (e.g. in where clauses or <code>fn</code> types), which is covered elsewhere in the <a href="./ty_module/binders.html">chapter on <code>Binder</code>s </a>.</p> |
| <h1 id="tygenerics"><a class="header" href="#tygenerics"><code>ty::Generics</code></a></h1> |
| <p>The generic parameters introduced by an item are tracked by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Generics.html"><code>ty::Generics</code></a> struct. Sometimes items allow usage of generics defined on parent items, this is accomplished via the <code>ty::Generics</code> struct having an optional field to specify a parent item to inherit generic parameters of. For example given the following code:</p> |
| <pre><code class="language-rust ignore">trait Trait<T> { |
| fn foo<U>(&self); |
| }</code></pre> |
| <p>The <code>ty::Generics</code> used for <code>foo</code> would contain <code>[U]</code> and a parent of <code>Some(Trait)</code>. <code>Trait</code> would have a <code>ty::Generics</code> containing <code>[Self, T]</code> with a parent of <code>None</code>.</p> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/generics/struct.GenericParamDef.html"><code>GenericParamDef</code></a> struct is used to represent each individual generic parameter in a <code>ty::Generics</code> listing. The <code>GenericParamDef</code> struct contains information about the generic parameter, for example its name, defid, what kind of parameter it is (i.e. type, const, lifetime).</p> |
| <p><code>GenericParamDef</code> also contains a <code>u32</code> index representing what position the parameter is (starting from the outermost parent), this is the value used to represent usages of generic parameters (more on this in the <a href="./ty.html">chapter on representing types</a>).</p> |
| <p>Interestingly, <code>ty::Generics</code> does not currently contain <em>every</em> generic parameter defined on an item. In the case of functions it only contains the <em>early bound</em> parameters.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="earlybinder-and-instantiating-parameters"><a class="header" href="#earlybinder-and-instantiating-parameters"><code>EarlyBinder</code> and instantiating parameters</a></h1> |
| <p>Given an item that introduces a generic parameter <code>T</code>, whenever we refer to types inside of <code>foo</code> (i.e. the return type or argument types) from outside of <code>foo</code> we must take care to handle the generic parameters defined on <code>foo</code>. As an example:</p> |
| <pre><code class="language-rust ignore">fn foo<T, U>(a: T, _b: U) -> T { a } |
| |
| fn main() { |
| let c = foo::<i32, u128>(1, 2); |
| }</code></pre> |
| <p>When type checking <code>main</code> we cannot just naively look at the return type of <code>foo</code> and assign the type <code>T</code> to the variable <code>c</code>, The function <code>main</code> does not define any generic parameters, <code>T</code> is completely meaningless in this context. More generally whenever an item introduces (binds) generic parameters, when accessing types inside the item from outside, the generic parameters must be instantiated with values from the outer item.</p> |
| <p>In rustc we track this via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.EarlyBinder.html"><code>EarlyBinder</code></a> type, the return type of <code>foo</code> is represented as an <code>EarlyBinder<Ty></code> with the only way to access <code>Ty</code> being to provide arguments for any generic parameters <code>Ty</code> might be using. This is implemented via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.EarlyBinder.html#method.instantiate"><code>EarlyBinder::instantiate</code></a> method which discharges the binder returning the inner value with all the generic parameters replaced by the provided arguments.</p> |
| <p>To go back to our example, when type checking <code>main</code> the return type of <code>foo</code> would be represented as <code>EarlyBinder(T/#0)</code>. Then, because we called the function with <code>i32, u128</code> for the generic arguments, we would call <code>EarlyBinder::instantiate</code> on the return type with <code>[i32, u128]</code> for the args. This would result in an instantiated return type of <code>i32</code> that we can use as the type of the local <code>c</code>.</p> |
| <p>Here are some more examples:</p> |
| <pre><code class="language-rust ignore">fn foo<T>() -> Vec<(u32, T)> { Vec::new() } |
| fn bar() { |
| // the return type of `foo` before instantiating it would be: |
| // `EarlyBinder(Adt(Vec, &[Tup(&[u32, T/#=0])]))` |
| // we then instantiate the binder with `[u64]` resulting in the type: |
| // `Adt(Vec, &[Tup(&[u32, u64])])` |
| let a = foo::<u64>(); |
| }</code></pre> |
| <pre><code class="language-rust ignore">struct Foo<A, B> { |
| x: Vec<A>, |
| .. |
| } |
| |
| fn bar(foo: Foo<u32, f32>) { |
| // the type of `foo`'s `x` field before instantiating it would be: |
| // `EarlyBinder(Vec<A/#0>)` |
| // we then instantiate the binder with `[u32, f32]` as those are the |
| // generic arguments to the `Foo` struct. This results in a type of: |
| // `Vec<u32>` |
| let y = foo.x; |
| }</code></pre> |
| <p>In the compiler the <code>instantiate</code> call for this is done in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.FieldDef.html#method.ty"><code>FieldDef::ty</code></a> (<a href="https://github.com/rust-lang/rust/blob/44d679b9021f03a79133021b94e6d23e9b55b3ab/compiler/rustc_middle/src/ty/mod.rs#L1421-L1426">src</a>), at some point during type checking <code>bar</code> we will wind up calling <code>FieldDef::ty(x, &[u32, f32])</code> in order to obtain the type of <code>foo.x</code>.</p> |
| <p><strong>Note on indices:</strong> It is a bug if the index of a <code>Param</code> does not match what the <code>EarlyBinder</code> binds. For |
| example, if the index is out of bounds or the index of a lifetime corresponds to a type parameter. |
| These sorts of errors are caught earlier in the compiler during name resolution where we disallow references |
| to generics parameters introduced by items that should not be nameable by the inner item.</p> |
| <hr /> |
| <p>As mentioned previously when <em>outside</em> of an item, it is important to instantiate the <code>EarlyBinder</code> with generic arguments before accessing the value inside, but the setup for when we are conceptually inside of the binder already is a bit different.</p> |
| <p>For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>impl<T> Trait for Vec<T> { |
| fn foo(&self, b: Self) {} |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>When constructing a <code>Ty</code> to represent the <code>b</code> parameter's type we need to get the type of <code>Self</code> on the impl that we are inside. This can be acquired by calling the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.type_of"><code>type_of</code></a> query with the <code>impl</code>'s <code>DefId</code>, however, this will return a <code>EarlyBinder<Ty></code> as the impl block binds generic parameters that may have to be discharged if we are outside of the impl.</p> |
| <p>The <code>EarlyBinder</code> type provides an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.EarlyBinder.html#method.instantiate_identity"><code>instantiate_identity</code></a> function for discharging the binder when you are "already inside of it". This is effectively a more performant version of writing <code>EarlyBinder::instantiate(GenericArgs::identity_for_item(..))</code>. Conceptually this discharges the binder by instantiating it with placeholders in the root universe (we will talk about what this means in the next few chapters). In practice though it simply returns the inner value with no modification taking place.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="binder-and-higher-ranked-regions"><a class="header" href="#binder-and-higher-ranked-regions"><code>Binder</code> and Higher ranked regions</a></h1> |
| <p>Sometimes we define generic parameters not on an item but as part of a type or a where clause. As an example the type <code>for<'a> fn(&'a u32)</code> or the where clause <code>for<'a> T: Trait<'a></code> both introduce a generic lifetime named <code>'a</code>. Currently there is no stable syntax for <code>for<T></code> or <code>for<const N: usize></code> but on nightly <code>feature(non_lifetime_binders)</code> can be used to write where clauses (but not types) using <code>for<T></code>/<code>for<const N: usize></code>.</p> |
| <p>The <code>for</code> is referred to as a "binder" because it brings new names into scope. In rustc we use the <code>Binder</code> type to track where these parameters are introduced and what the parameters are (i.e. how many and whether the parameter is a type/const/region). A type such as <code>for<'a> fn(&'a u32)</code> would be |
| represented in rustc as:</p> |
| <pre><code>Binder( |
| fn(&RegionKind::Bound(DebruijnIndex(0), BoundVar(0)) u32) -> (), |
| &[BoundVariableKind::Region(...)], |
| ) |
| </code></pre> |
| <p>Usages of these parameters is represented by the <code>RegionKind::Bound</code> (or <code>TyKind::Bound</code>/<code>ConstKind::Bound</code> variants). These bound regions/types/consts are composed of two main pieces of data:</p> |
| <ul> |
| <li>A <a href="ty_module/../appendix/background.html#what-is-a-de-bruijn-index">DebruijnIndex</a> to specify which binder we are referring to.</li> |
| <li>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.BoundVar.html"><code>BoundVar</code></a> which specifies which of the parameters that the <code>Binder</code> introduces we are referring to.</li> |
| </ul> |
| <p>We also sometimes store some extra information for diagnostics reasons via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.BoundTyKind.html"><code>BoundTyKind</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.BoundRegionKind.html"><code>BoundRegionKind</code></a> but this is not important for type equality or more generally the semantics of <code>Ty</code>. (omitted from the above example)</p> |
| <p>In debug output (and also informally when talking to each other) we tend to write these bound variables in the format of <code>^DebruijnIndex_BoundVar</code>. The above example would instead be written as <code>Binder(fn(&'^0_0), &[BoundVariableKind::Region])</code>. Sometimes when the <code>DebruijnIndex</code> is <code>0</code> we just omit it and would write <code>^0</code>.</p> |
| <p>Another concrete example, this time a mixture of <code>for<'a></code> in a where clause and a type:</p> |
| <pre><code>where |
| for<'a> Foo<for<'b> fn(&'a &'b T)>: Trait, |
| </code></pre> |
| <p>This would be represented as</p> |
| <pre><code>Binder( |
| Foo<Binder( |
| fn(&'^1_0 &'^0 T/#0), |
| [BoundVariableKind::Region(...)] |
| )>: Trait, |
| [BoundVariableKind::Region(...)] |
| ) |
| </code></pre> |
| <p>Note how the <code>'^1_0</code> refers to the <code>'a</code> parameter. We use a <code>DebruijnIndex</code> of <code>1</code> to refer to the binder one level up from the innermost one, and a var of <code>0</code> to refer to the first parameter bound which is <code>'a</code>. We also use <code>'^0</code> to refer to the <code>'b</code> parameter, the <code>DebruijnIndex</code> is <code>0</code> (referring to the innermost binder) so we omit it, leaving only the boundvar of <code>0</code> referring to the first parameter bound which is <code>'b</code>.</p> |
| <p>We did not always explicitly track the set of bound vars introduced by each <code>Binder</code>, this caused a number of bugs (read: ICEs <a href="https://github.com/rust-lang/rust/issues/81193">#81193</a>, <a href="https://github.com/rust-lang/rust/issues/79949">#79949</a>, <a href="https://github.com/rust-lang/rust/issues/83017">#83017</a>). By tracking these explicitly we can assert when constructing higher ranked where clauses/types that there are no escaping bound variables or variables from a different binder. See the following example of an invalid type inside of a binder:</p> |
| <pre><code>Binder( |
| fn(&'^1_0 &'^1 T/#0), |
| &[BoundVariableKind::Region(...)], |
| ) |
| </code></pre> |
| <p>This would cause all kinds of issues as the region <code>'^1_0</code> refers to a binder at a higher level than the outermost binder i.e. it is an escaping bound var. The <code>'^1</code> region (also writeable as <code>'^0_1</code>) is also ill formed as the binder it refers to does not introduce a second parameter. Modern day rustc will ICE when constructing this binder due to both of those reasons, in the past we would have simply allowed this to work and then ran into issues in other parts of the codebase.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="instantiating-binders"><a class="header" href="#instantiating-binders">Instantiating <code>Binder</code>s</a></h1> |
| <p>Much like <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.EarlyBinder.html"><code>EarlyBinder</code></a>, when accessing the inside of a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.Binder.html"><code>Binder</code></a> we must first discharge it by replacing the bound vars with some other value. This is for much the same reason as with <code>EarlyBinder</code>, types referencing parameters introduced by the <code>Binder</code> do not make any sense outside of that binder, for example:</p> |
| <pre><code class="language-rust ignore">fn foo<'a>(a: &'a u32) -> &'a u32 { |
| a |
| } |
| fn bar<T>(a: fn(&u32) -> T) -> T { |
| a(&10) |
| } |
| |
| fn main() { |
| let higher_ranked_fn_ptr = foo as for<'a> fn(&'a u32) -> &'a u32; |
| let references_bound_vars = bar(higher_ranked_fn_ptr); |
| }</code></pre> |
| <p>In this example we are providing an argument of type <code>for<'a> fn(&'^0 u32) -> &'^0 u32</code> to <code>bar</code>, we do not want to allow <code>T</code> to be inferred to the type <code>&'^0 u32</code> as it would be rather nonsensical (and likely unsound if we did not happen to ICE, <code>main</code> has no idea what <code>'a</code> is so how would the borrow checker handle a borrow with lifetime <code>'a</code>).</p> |
| <p>Unlike <code>EarlyBinder</code> we typically do not instantiate <code>Binder</code> with some concrete set of arguments from the user, i.e. <code>['b, 'static]</code> as arguments to a <code>for<'a1, 'a2> fn(&'a1 u32, &'a2 u32)</code>. Instead we usually instantiate the binder with inference variables or placeholders.</p> |
| <h2 id="instantiating-with-inference-variables"><a class="header" href="#instantiating-with-inference-variables">Instantiating with inference variables</a></h2> |
| <p>We instantiate binders with inference variables when we are trying to infer a possible instantiation of the binder, e.g. calling higher ranked function pointers or attempting to use a higher ranked where-clause to prove some bound. For example, given the <code>higher_ranked_fn_ptr</code> from the example above, if we were to call it with <code>&10_u32</code> we would:</p> |
| <ul> |
| <li>Instantiate the binder with infer vars yielding a signature of <code>fn(&'?0 u32) -> &'?0 u32)</code></li> |
| <li>Equate the type of the provided argument <code>&10_u32</code> (&'static u32) with the type in the signature, <code>&'?0 u32</code>, inferring <code>'?0 = 'static</code></li> |
| <li>The provided arguments were correct as we were successfully able to unify the types of the provided arguments with the types of the arguments in fn ptr signature</li> |
| </ul> |
| <p>As another example of instantiating with infer vars, given some <code>for<'a> T: Trait<'a></code> where-clause, if we were attempting to prove that <code>T: Trait<'static></code> holds we would:</p> |
| <ul> |
| <li>Instantiate the binder with infer vars yielding a where clause of <code>T: Trait<'?0></code></li> |
| <li>Equate the goal of <code>T: Trait<'static></code> with the instantiated where clause, inferring <code>'?0 = 'static</code></li> |
| <li>The goal holds because we were successfully able to unify <code>T: Trait<'static></code> with <code>T: Trait<'?0></code></li> |
| </ul> |
| <p>Instantiating binders with inference variables can be accomplished by using the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/struct.InferCtxt.html#method.instantiate_binder_with_fresh_vars"><code>instantiate_binder_with_fresh_vars</code></a> method on <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/struct.InferCtxt.html"><code>InferCtxt</code></a>. Binders should be instantiated with infer vars when we only care about one specific instantiation of the binder, if instead we wish to reason about all possible instantiations of the binder then placeholders should be used instead.</p> |
| <h2 id="instantiating-with-placeholders"><a class="header" href="#instantiating-with-placeholders">Instantiating with placeholders</a></h2> |
| <p>Placeholders are very similar to <code>Ty/ConstKind::Param</code>/<code>ReEarlyParam</code>, they represent some unknown type that is only equal to itself. <code>Ty</code>/<code>Const</code> and <code>Region</code> all have a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Placeholder.html"><code>Placeholder</code></a> variant that is comprised of a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.UniverseIndex.html"><code>Universe</code></a> and a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.BoundVar.html"><code>BoundVar</code></a>.</p> |
| <p>The <code>Universe</code> tracks which binder the placeholder originated from, and the <code>BoundVar</code> tracks which parameter on said binder that this placeholder corresponds to. Equality of placeholders is determined solely by whether the universes are equal and the <code>BoundVar</code>s are equal. See the <a href="ty_module/../borrow_check/region_inference/placeholders_and_universes.html">chapter on Placeholders and Universes</a> for more information.</p> |
| <p>When talking with other rustc devs or seeing <code>Debug</code> formatted <code>Ty</code>/<code>Const</code>/<code>Region</code>s, <code>Placeholder</code> will often be written as <code>'!UNIVERSE_BOUNDVARS</code>. For example given some type <code>for<'a> fn(&'a u32, for<'b> fn(&'b &'a u32))</code>, after instantiating both binders (assuming the <code>Universe</code> in the current <code>InferCtxt</code> was <code>U0</code> beforehand), the type of <code>&'b &'a u32</code> would be represented as <code>&'!2_0 &!1_0 u32</code>.</p> |
| <p>When the universe of the placeholder is <code>0</code>, it will be entirely omitted from the debug output, i.e. <code>!0_2</code> would be printed as <code>!2</code>. This rarely happens in practice though as we increase the universe in the <code>InferCtxt</code> when instantiating a binder with placeholders so usually the lowest universe placeholders encounterable are ones in <code>U1</code>.</p> |
| <p><code>Binder</code>s can be instantiated with placeholders via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/struct.InferCtxt.html#method.enter_forall"><code>enter_forall</code></a> method on <code>InferCtxt</code>. It should be used whenever the compiler should care about any possible instantiation of the binder instead of one concrete instantiation.</p> |
| <p>Note: in the original example of this chapter it was mentioned that we should not infer a local variable to have type <code>&'^0 u32</code>. This code is prevented from compiling via universes (as explained in the linked chapter)</p> |
| <h3 id="why-have-both-replaceholder-and-rebound"><a class="header" href="#why-have-both-replaceholder-and-rebound">Why have both <code>RePlaceholder</code> and <code>ReBound</code>?</a></h3> |
| <p>You may be wondering why we have both of these variants, afterall the data stored in <code>Placeholder</code> is effectively equivalent to that of <code>ReBound</code>: something to track which binder, and an index to track which parameter the <code>Binder</code> introduced.</p> |
| <p>The main reason for this is that <code>Bound</code> is a more syntactic representation of bound variables whereas <code>Placeholder</code> is a more semantic representation. As a concrete example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>impl<'a> Other<'a> for &'a u32 { } |
| |
| impl<T> Trait for T |
| where |
| for<'a> T: Other<'a>, |
| { ... } |
| |
| impl<T> Bar for T |
| where |
| for<'a> &'a T: Trait |
| { ... } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Given these trait implementations <code>u32: Bar</code> should <em>not</em> hold. <code>&'a u32</code> only implements <code>Other<'a></code> when the lifetime of the borrow and the lifetime on the trait are equal. However if we only used <code>ReBound</code> and did not have placeholders it may be easy to accidentally believe that trait bound does hold. To explain this let's walk through an example of trying to prove <code>u32: Bar</code> in a world where rustc did not have placeholders:</p> |
| <ul> |
| <li>We start by trying to prove <code>u32: Bar</code></li> |
| <li>We find the <code>impl<T> Bar for T</code> impl, we would wind up instantiating the <code>EarlyBinder</code> with <code>u32</code> (note: this is not <em>quite</em> accurate as we first instantiate the binder with an inference variable that we then infer to be <code>u32</code> but that distinction is not super important here)</li> |
| <li>There is a where clause <code>for<'a> &'^0 T: Trait</code> on the impl, as we instantiated the early binder with <code>u32</code> we actually have to prove <code>for<'a> &'^0 u32: Trait</code></li> |
| <li>We find the <code>impl<T> Trait for T</code> impl, we would wind up instantiating the <code>EarlyBinder</code> with <code>&'^0 u32</code></li> |
| <li>There is a where clause <code>for<'a> T: Other<'^0></code>, as we instantiated the early binder with <code>&'^0 u32</code> we actually have to prove <code>for<'a> &'^0 u32: Other<'^0></code></li> |
| <li>We find the <code>impl<'a> Other<'a> for &'a u32</code> and this impl is enough to prove the bound as the lifetime on the borrow and on the trait are both <code>'^0</code></li> |
| </ul> |
| <p>This end result is incorrect as we had two separate binders introducing their own generic parameters, the trait bound should have ended up as something like <code>for<'a1, 'a2> &'^1 u32: Other<'^0></code> which is <em>not</em> satisfied by the <code>impl<'a> Other<'a> for &'a u32</code>.</p> |
| <p>While in theory we could make this work it would be quite involved and more complex than the current setup, we would have to:</p> |
| <ul> |
| <li>"rewrite" bound variables to have a higher <code>DebruijnIndex</code> whenever instantiating a <code>Binder</code>/<code>EarlyBinder</code> with a <code>Bound</code> ty/const/region</li> |
| <li>When inferring an inference variable to a bound var, if that bound var is from a binder entered after creating the infer var, we would have to lower the <code>DebruijnIndex</code> of the var.</li> |
| <li>Separately track what binder an inference variable was created inside of, also what the innermost binder it can name parameters from (currently we only have to track the latter)</li> |
| <li>When resolving inference variables rewrite any bound variables according to the current binder depth of the infcx</li> |
| <li>Maybe more (while writing this list items kept getting added so it seems naive to think this is exhaustive)</li> |
| </ul> |
| <p>Fundamentally all of this complexity is because <code>Bound</code> ty/const/regions have a different representation for a given parameter on a <code>Binder</code> depending on how many other <code>Binder</code>s there are between the binder introducing the parameter, and its usage. For example given the following code:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T>() |
| where |
| for<'a> T: Trait<'a, for<'b> fn(&'b T, &'a u32)> |
| { ... } |
| <span class="boring">}</span></code></pre></pre> |
| <p>That where clause would be written as:<br /> |
| <code>for<'a> T: Trait<'^0, for<'b> fn(&'^0 T, &'^1_0 u32)></code><br /> |
| Despite there being two references to the <code>'a</code> parameter they are both represented differently: <code>^0</code> and <code>^1_0</code>, due to the fact that the latter usage is nested under a second <code>Binder</code> for the inner function pointer type.</p> |
| <p>This is in contrast to <code>Placeholder</code> ty/const/regions which do not have this limitation due to the fact that <code>Universe</code>s are specific to the current <code>InferCtxt</code> not the usage site of the parameter.</p> |
| <p>It is trivially possible to instantiate <code>EarlyBinder</code>s and unify inference variables with existing <code>Placeholder</code>s as no matter what context the <code>Placeholder</code> is in, it will have the same representation. As an example if we were to instantiate the binder on the higher ranked where clause from above, it would be represented like so:<br /> |
| <code>T: Trait<'!1_0, for<'b> fn(&'^0 T, &'!1_0 u32)></code><br /> |
| the <code>RePlaceholder</code> representation for both usages of <code>'a</code> are the same despite one being underneath another <code>Binder</code>.</p> |
| <p>If we were to then instantiate the binder on the function pointer we would get a type such as:<br /> |
| <code>fn(&'!2_0 T, ^'!1_0 u32)</code><br /> |
| the <code>RePlaceholder</code> for the <code>'b</code> parameter is in a higher universe to track the fact that its binder was instantiated after the binder for <code>'a</code>.</p> |
| <h2 id="instantiating-with-relateparam"><a class="header" href="#instantiating-with-relateparam">Instantiating with <code>ReLateParam</code></a></h2> |
| <p>As discussed in <a href="ty_module/param_ty_const_regions.html">the chapter about representing types</a>, <code>RegionKind</code> has two variants for representing generic parameters, <code>ReLateParam</code> and <code>ReEarlyParam</code>. |
| <code>ReLateParam</code> is conceptually a <code>Placeholder</code> that is always in the root universe (<code>U0</code>). It is used when instantiating late bound parameters of functions/closures while inside of them. Its actual representation is relatively different from both <code>ReEarlyParam</code> and <code>RePlaceholder</code>:</p> |
| <ul> |
| <li>A <code>DefId</code> for the item that introduced the late bound generic parameter</li> |
| <li>A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.BoundRegionKind.html"><code>BoundRegionKind</code></a> which either specifies the <code>DefId</code> of the generic parameter and its name (via a <code>Symbol</code>), or that this placeholder is representing the anonymous lifetime of a <code>Fn</code>/<code>FnMut</code> closure's self borrow. There is also a variant for <code>BrAnon</code> but this is not used for <code>ReLateParam</code>.</li> |
| </ul> |
| <p>For example, given the following code:</p> |
| <pre><code class="language-rust ignore">impl Trait for Whatever { |
| fn foo<'a>(a: &'a u32) -> &'a u32 { |
| let b: &'a u32 = a; |
| b |
| } |
| }</code></pre> |
| <p>the lifetime <code>'a</code> in the type <code>&'a u32</code> in the function body would be represented as:</p> |
| <pre><code>ReLateParam( |
| {impl#0}::foo, |
| BoundRegionKind::BrNamed({impl#0}::foo::'a, "'a") |
| ) |
| </code></pre> |
| <p>In this specific case of referencing late bound generic parameters of a function from inside the body this is done implicitly during <code>hir_ty_lowering</code> rather than explicitly when instantiating a <code>Binder</code> somewhere. In some cases however, we do explicitly instantiate a <code>Binder</code> with <code>ReLateParam</code>s.</p> |
| <p>Generally whenever we have a <code>Binder</code> for late bound parameters on a function/closure and we are conceptually inside of the binder already, we use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.liberate_late_bound_regions"><code>liberate_late_bound_regions</code></a> to instantiate it with <code>ReLateParam</code>s. That makes this operation the <code>Binder</code> equivalent to <code>EarlyBinder</code>'s <code>instantiate_identity</code>.</p> |
| <p>As a concrete example, accessing the signature of a function we are type checking will be represented as <code>EarlyBinder<Binder<FnSig>></code>. As we are already "inside" of these binders, we would call <code>instantiate_identity</code> followed by <code>liberate_late_bound_regions</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="early-vs-late-bound-parameters"><a class="header" href="#early-vs-late-bound-parameters">Early vs Late bound parameters</a></h1> |
| <blockquote> |
| <p><strong>NOTE</strong>: This chapter largely talks about early/late bound as being solely relevant when discussing function item types/function definitions. This is potentially not completely true, async blocks and closures should likely be discussed somewhat in this chapter.</p> |
| </blockquote> |
| <h2 id="what-does-it-mean-to-be-early-bound-or-late-bound"><a class="header" href="#what-does-it-mean-to-be-early-bound-or-late-bound">What does it mean to be "early" bound or "late" bound</a></h2> |
| <p>Every function definition has a corresponding ZST that implements the <code>Fn*</code> traits known as a <a href="https://doc.rust-lang.org/reference/types/function-item.html">function item type</a>. This part of the chapter will talk a little bit about the "desugaring" of function item types as it is useful context for explaining the difference between early bound and late bound generic parameters.</p> |
| <p>Let's start with a very trivial example involving no generic parameters:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo(a: String) -> u8 { |
| <span class="boring"> 1 |
| </span> /* snip */ |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>If we explicitly wrote out the definitions for the function item type corresponding to <code>foo</code> and its associated <code>Fn</code> impl it would look something like this:</p> |
| <pre><code class="language-rust ignore">struct FooFnItem; |
| |
| impl Fn<(String,)> for FooFnItem { |
| type Output = u8; |
| /* fn call(&self, ...) -> ... { ... } */ |
| }</code></pre> |
| <p>The builtin impls for the <code>FnMut</code>/<code>FnOnce</code> traits as well as the impls for <code>Copy</code> and <code>Clone</code> were omitted for brevity reasons (although these traits <em>are</em> implemented for function item types).</p> |
| <p>A slightly more complicated example would involve introducing generic parameters to the function:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T: Sized>(a: T) -> T { |
| <span class="boring"> a |
| </span> /* snip */ |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Writing out the definitions would look something like this:</p> |
| <pre><code class="language-rust ignore">struct FooFnItem<T: Sized>(PhantomData<fn(T) -> T>); |
| |
| impl<T: Sized> Fn<(T,)> for FooFnItem<T> { |
| type Output = T; |
| /* fn call(&self, ...) -> ... { ... } */ |
| }</code></pre> |
| <p>Note that the function item type <code>FooFnItem</code> is generic over some type parameter <code>T</code> as defined on the function <code>foo</code>. However, not all generic parameters defined on functions are also defined on the function item type as demonstrated here:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a, T: Sized>(a: &'a T) -> &'a T { |
| <span class="boring"> a |
| </span> /* snip */ |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>With its "desugared" form looking like so:</p> |
| <pre><code class="language-rust ignore">struct FooFnItem<T: Sized>(PhantomData<for<'a> fn(&'a T) -> &'a T>); |
| |
| impl<'a, T: Sized> Fn<(&'a T,)> for FooFnItem<T> { |
| type Output = &'a T; |
| /* fn call(&self, ...) -> ... { ... } */ |
| }</code></pre> |
| <p>The lifetime parameter <code>'a</code> from the function <code>foo</code> is not present on the function item type <code>FooFnItem</code> and is instead introduced on the builtin impl solely for use in representing the argument types.</p> |
| <p>Generic parameters not all being defined on the function item type means that there are two steps where generic arguments are provided when calling a function.</p> |
| <ol> |
| <li>Naming the function (e.g. <code>let a = foo;</code>) the arguments for <code>FooFnItem</code> are provided.</li> |
| <li>Calling the function (e.g. <code>a(&10);</code>) any parameters defined on the builtin impl are provided.</li> |
| </ol> |
| <p>This two-step system is where the early vs late naming scheme comes from, early bound parameters are provided in the <em>earliest</em> step (naming the function), whereas late bound parameters are provided in the <em>latest</em> step (calling the function).</p> |
| <p>Looking at the desugaring from the previous example we can tell that <code>T</code> is an early bound type parameter and <code>'a</code> is a late bound lifetime parameter as <code>T</code> is present on the function item type but <code>'a</code> is not. See this example of calling <code>foo</code> annotated with where each generic parameter has an argument provided:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a, T: Sized>(a: &'a T) -> &'a T { |
| <span class="boring"> a |
| </span> /* snip */ |
| } |
| |
| // Here we provide a type argument `String` to the |
| // type parameter `T` on the function item type |
| let my_func = foo::<String>; |
| |
| // Here (implicitly) a lifetime argument is provided |
| // to the lifetime parameter `'a` on the builtin impl. |
| my_func(&String::new()); |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="differences-between-early-and-late-bound-parameters"><a class="header" href="#differences-between-early-and-late-bound-parameters">Differences between early and late bound parameters</a></h2> |
| <h3 id="higher-ranked-function-pointers-and-trait-bounds"><a class="header" href="#higher-ranked-function-pointers-and-trait-bounds">Higher ranked function pointers and trait bounds</a></h3> |
| <p>A generic parameter being late bound allows for more flexible usage of the function item. For example if we have some function <code>foo</code> with an early bound lifetime parameter and some function <code>bar</code> with a late bound lifetime parameter <code>'a</code> we would have the following builtin <code>Fn</code> impls:</p> |
| <pre><code class="language-rust ignore">impl<'a> Fn<(&'a String,)> for FooFnItem<'a> { /* ... */ } |
| impl<'a> Fn<(&'a String,)> for BarFnItem { /* ... */ }</code></pre> |
| <p>The <code>bar</code> function has a strictly more flexible signature as the function item type can be called with a borrow with <em>any</em> lifetime, whereas the <code>foo</code> function item type would only be callable with a borrow with the same lifetime on the function item type. We can show this by simply trying to call <code>foo</code>'s function item type multiple times with different lifetimes:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// The `'a: 'a` bound forces this lifetime to be early bound. |
| fn foo<'a: 'a>(b: &'a String) -> &'a String { b } |
| fn bar<'a>(b: &'a String) -> &'a String { b } |
| |
| // Early bound generic parameters are instantiated here when naming |
| // the function `foo`. As `'a` is early bound an argument is provided. |
| let f = foo::<'_>; |
| |
| // Both function arguments are required to have the same lifetime as |
| // the lifetime parameter being early bound means that `f` is only |
| // callable for one specific lifetime. |
| // |
| // As we call this with borrows of different lifetimes, the borrow checker |
| // will error here. |
| f(&String::new()); |
| f(&String::new()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example we call <code>foo</code>'s function item type twice, each time with a borrow of a temporary. These two borrows could not possible have lifetimes that overlap as the temporaries are only alive during the function call, not after. The lifetime parameter on <code>foo</code> being early bound requires all callers of <code>f</code> to provide a borrow with the same lifetime, as this is not possible the borrow checker errors.</p> |
| <p>If the lifetime parameter on <code>foo</code> was late bound this would be able to compile as each caller could provide a different lifetime argument for its borrow. See the following example which demonstrates this using the <code>bar</code> function defined above:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span><span class="boring">fn foo<'a: 'a>(b: &'a String) -> &'a String { b } |
| </span><span class="boring">fn bar<'a>(b: &'a String) -> &'a String { b } |
| </span><span class="boring"> |
| </span>// Early bound parameters are instantiated here, however as `'a` is |
| // late bound it is not provided here. |
| let b = bar; |
| |
| // Late bound parameters are instantiated separately at each call site |
| // allowing different lifetimes to be used by each caller. |
| b(&String::new()); |
| b(&String::new()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>This is reflected in the ability to coerce function item types to higher ranked function pointers and prove higher ranked <code>Fn</code> trait bounds. We can demonstrate this with the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// The `'a: 'a` bound forces this lifetime to be early bound. |
| fn foo<'a: 'a>(b: &'a String) -> &'a String { b } |
| fn bar<'a>(b: &'a String) -> &'a String { b } |
| |
| fn accepts_hr_fn(_: impl for<'a> Fn(&'a String) -> &'a String) {} |
| |
| fn higher_ranked_trait_bound() { |
| let bar_fn_item = bar; |
| accepts_hr_fn(bar_fn_item); |
| |
| let foo_fn_item = foo::<'_>; |
| // errors |
| accepts_hr_fn(foo_fn_item); |
| } |
| |
| fn higher_ranked_fn_ptr() { |
| let bar_fn_item = bar; |
| let fn_ptr: for<'a> fn(&'a String) -> &'a String = bar_fn_item; |
| |
| let foo_fn_item = foo::<'_>; |
| // errors |
| let fn_ptr: for<'a> fn(&'a String) -> &'a String = foo_fn_item; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>In both of these cases the borrow checker errors as it does not consider <code>foo_fn_item</code> to be callable with a borrow of any lifetime. This is due to the fact that the lifetime parameter on <code>foo</code> is early bound, causing <code>foo_fn_item</code> to have a type of <code>FooFnItem<'_></code> which (as demonstrated by the desugared <code>Fn</code> impl) is only callable with a borrow of the same lifetime <code>'_</code>.</p> |
| <h3 id="turbofishing-in-the-presence-of-late-bound-parameters"><a class="header" href="#turbofishing-in-the-presence-of-late-bound-parameters">Turbofishing in the presence of late bound parameters</a></h3> |
| <p>As mentioned previously, the distinction between early and late bound parameters means that there are two places where generic parameters are instantiated:</p> |
| <ul> |
| <li>When naming a function (early)</li> |
| <li>When calling a function (late)</li> |
| </ul> |
| <p>There is currently no syntax for explicitly specifying generic arguments for late bound parameters during the call step; generic arguments can only be specified for early bound parameters when naming a function. |
| The syntax <code>foo::<'static>();</code>, despite being part of a function call, behaves as <code>(foo::<'static>)();</code> and instantiates the early bound generic parameters on the function item type.</p> |
| <p>See the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a>(b: &'a u32) -> &'a u32 { b } |
| |
| let f /* : FooFnItem<????> */ = foo::<'static>; |
| <span class="boring">}</span></code></pre></pre> |
| <p>The above example errors as the lifetime parameter <code>'a</code> is late bound and so cannot be instantiated as part of the "naming a function" step. If we make the lifetime parameter early bound we will see this code start to compile:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a: 'a>(b: &'a u32) -> &'a u32 { b } |
| |
| let f /* : FooFnItem<'static> */ = foo::<'static>; |
| <span class="boring">}</span></code></pre></pre> |
| <p>What the current implementation of the compiler aims to do is error when specifying lifetime arguments to a function that has both early <em>and</em> late bound lifetime parameters. In practice, due to excessive breakage, some cases are actually only future compatibility warnings (<a href="https://github.com/rust-lang/rust/issues/42868">#42868</a>):</p> |
| <ul> |
| <li>When the amount of lifetime arguments is the same as the number of early bound lifetime parameters a FCW is emitted instead of an error</li> |
| <li>An error is always downgraded to a FCW when using method call syntax</li> |
| </ul> |
| <p>To demonstrate this we can write out the different kinds of functions and give them both a late and early bound lifetime:</p> |
| <pre><code class="language-rust ignore">fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| |
| struct Foo; |
| |
| trait Trait: Sized { |
| fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); |
| fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); |
| } |
| |
| impl Trait for Foo { |
| fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} |
| fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| } |
| |
| impl Foo { |
| fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} |
| fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| }</code></pre> |
| <p>Then, for the first case, we can call each function with a single lifetime argument (corresponding to the one early bound lifetime parameter) and note that it only results in a FCW rather than a hard error.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![deny(late_bound_lifetime_arguments)] |
| |
| <span class="boring">fn main() { |
| </span><span class="boring">fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| </span><span class="boring"> |
| </span><span class="boring">struct Foo; |
| </span><span class="boring"> |
| </span><span class="boring">trait Trait: Sized { |
| </span><span class="boring"> fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); |
| </span><span class="boring"> fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); |
| </span><span class="boring">} |
| </span><span class="boring"> |
| </span><span class="boring">impl Trait for Foo { |
| </span><span class="boring"> fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} |
| </span><span class="boring"> fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| </span><span class="boring">} |
| </span><span class="boring"> |
| </span><span class="boring">impl Foo { |
| </span><span class="boring"> fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} |
| </span><span class="boring"> fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| </span><span class="boring">} |
| </span><span class="boring"> |
| </span>// Specifying as many arguments as there are early |
| // bound parameters is always a future compat warning |
| Foo.trait_method::<'static>(&(), &()); |
| Foo::trait_method::<'static>(Foo, &(), &()); |
| Foo::trait_function::<'static>(&(), &()); |
| Foo.inherent_method::<'static>(&(), &()); |
| Foo::inherent_function::<'static>(&(), &()); |
| free_function::<'static>(&(), &()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>For the second case we call each function with more lifetime arguments than there are lifetime parameters (be it early or late bound) and note that method calls result in a FCW as opposed to the free/associated functions which result in a hard error:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span><span class="boring">fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| </span><span class="boring"> |
| </span><span class="boring">struct Foo; |
| </span><span class="boring"> |
| </span><span class="boring">trait Trait: Sized { |
| </span><span class="boring"> fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); |
| </span><span class="boring"> fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); |
| </span><span class="boring">} |
| </span><span class="boring"> |
| </span><span class="boring">impl Trait for Foo { |
| </span><span class="boring"> fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} |
| </span><span class="boring"> fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| </span><span class="boring">} |
| </span><span class="boring"> |
| </span><span class="boring">impl Foo { |
| </span><span class="boring"> fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} |
| </span><span class="boring"> fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} |
| </span><span class="boring">} |
| </span><span class="boring"> |
| </span>// Specifying more arguments than there are early |
| // bound parameters is a future compat warning when |
| // using method call syntax. |
| Foo.trait_method::<'static, 'static, 'static>(&(), &()); |
| Foo.inherent_method::<'static, 'static, 'static>(&(), &()); |
| // However, it is a hard error when not using method call syntax. |
| Foo::trait_method::<'static, 'static, 'static>(Foo, &(), &()); |
| Foo::trait_function::<'static, 'static, 'static>(&(), &()); |
| Foo::inherent_function::<'static, 'static, 'static>(&(), &()); |
| free_function::<'static, 'static, 'static>(&(), &()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>Even when specifying enough lifetime arguments for both the late and early bound lifetime parameter, these arguments are not actually used to annotate the lifetime provided to late bound parameters. We can demonstrate this by turbofishing <code>'static</code> to a function while providing a non-static borrow:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct Foo; |
| |
| impl Foo { |
| fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b String ) {} |
| } |
| |
| Foo.inherent_method::<'static, 'static>(&(), &String::new()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>This compiles even though the <code>&String::new()</code> function argument does not have a <code>'static</code> lifetime, this is because "extra" lifetime arguments are discarded rather than taken into account for late bound parameters when actually calling the function.</p> |
| <h3 id="liveness-of-types-with-late-bound-parameters"><a class="header" href="#liveness-of-types-with-late-bound-parameters">Liveness of types with late bound parameters</a></h3> |
| <p>When checking type outlives bounds involving function item types we take into account early bound parameters. For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T>(_: T) {} |
| |
| fn requires_static<T: 'static>(_: T) {} |
| |
| fn bar<T>() { |
| let f /* : FooFnItem<T> */ = foo::<T>; |
| requires_static(f); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>As the type parameter <code>T</code> is early bound, the desugaring of the function item type for <code>foo</code> would look something like <code>struct FooFnItem<T></code>. Then in order for <code>FooFnItem<T>: 'static</code> to hold we must also require <code>T: 'static</code> to hold as otherwise we would wind up with soundness bugs.</p> |
| <p>Unfortunately, due to bugs in the compiler, we do not take into account early bound <em>lifetimes</em>, which is the cause of the open soundness bug <a href="https://github.com/rust-lang/rust/issues/84366">#84366</a>. This means that it's impossible to demonstrate a "difference" between early/late bound parameters for liveness/type outlives bounds as the only kind of generic parameters that are able to be late bound are lifetimes which are handled incorrectly.</p> |
| <p>Regardless, in theory the code example below <em>should</em> demonstrate such a difference once <a href="https://github.com/rust-lang/rust/issues/84366">#84366</a> is fixed:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn early_bound<'a: 'a>(_: &'a String) {} |
| fn late_bound<'a>(_: &'a String) {} |
| |
| fn requires_static<T: 'static>(_: T) {} |
| |
| fn bar<'b>() { |
| let e = early_bound::<'b>; |
| // this *should* error but does not |
| requires_static(e); |
| |
| let l = late_bound; |
| // this correctly does not error |
| requires_static(l); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="requirements-for-a-parameter-to-be-late-bound"><a class="header" href="#requirements-for-a-parameter-to-be-late-bound">Requirements for a parameter to be late bound</a></h2> |
| <h3 id="must-be-a-lifetime-parameter"><a class="header" href="#must-be-a-lifetime-parameter">Must be a lifetime parameter</a></h3> |
| <p>Type and Const parameters are not able to be late bound as we do not have a way to support types such as <code>dyn for<T> Fn(Box<T>)</code> or <code>for<T> fn(Box<T>)</code>. Calling such types requires being able to monomorphize the underlying function which is not possible with indirection through dynamic dispatch.</p> |
| <h3 id="must-not-be-used-in-a-where-clause"><a class="header" href="#must-not-be-used-in-a-where-clause">Must not be used in a where clause</a></h3> |
| <p>Currently when a generic parameter is used in a where clause it must be early bound. For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span><span class="boring">trait Trait<'a> {} |
| </span>fn foo<'a, T: Trait<'a>>(_: &'a String, _: T) {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example the lifetime parameter <code>'a</code> is considered to be early bound as it appears in the where clause <code>T: Trait<'a></code>. This is true even for "trivial" where clauses such as <code>'a: 'a</code> or those implied by wellformedness of function arguments, for example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a: 'a>(_: &'a String) {} |
| fn bar<'a, T: 'a>(_: &'a T) {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>In both of these functions the lifetime parameter <code>'a</code> would be considered to be early bound even though the where clauses they are used in arguably do not actually impose any constraints on the caller.</p> |
| <p>The reason for this restriction is a combination of two things:</p> |
| <ul> |
| <li>We cannot prove bounds on late bound parameters until they have been instantiated</li> |
| <li>Function pointers and trait objects do not have a way to represent yet to be proven where clauses from the underlying function</li> |
| </ul> |
| <p>Take the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<'a> {} |
| fn foo<'a, T: Trait<'a>>(_: &'a T) {} |
| |
| let f = foo::<String>; |
| let f = f as for<'a> fn(&'a String); |
| f(&String::new()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>At <em>some point</em> during type checking an error should be emitted for this code as <code>String</code> does not implement <code>Trait</code> for any lifetime.</p> |
| <p>If the lifetime <code>'a</code> were late bound then this becomes difficult to check. When naming <code>foo</code> we do not know what lifetime should be used as part of the <code>T: Trait<'a></code> trait bound as it has not yet been instantiated. When coercing the function item type to a function pointer we have no way of tracking the <code>String: Trait<'a></code> trait bound that must be proven when calling the function.</p> |
| <p>If the lifetime <code>'a</code> is early bound (which it is in the current implementation in rustc), then the trait bound can be checked when naming the function <code>foo</code>. Requiring parameters used in where clauses to be early bound gives a natural place to check where clauses defined on the function.</p> |
| <p>Finally, we do not require lifetimes to be early bound if they are used in <em>implied bounds</em>, for example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a, T>(_: &'a T) {} |
| |
| let f = foo; |
| f(&String::new()); |
| f(&String::new()); |
| <span class="boring">}</span></code></pre></pre> |
| <p>This code compiles, demonstrating that the lifetime parameter is late bound, even though <code>'a</code> is used in the type <code>&'a T</code> which implicitly requires <code>T: 'a</code> to hold. Implied bounds can be treated specially as any types introducing implied bounds are in the signature of the function pointer type, which means that when calling the function we know to prove <code>T: 'a</code>.</p> |
| <h3 id="must-be-constrained-by-argument-types"><a class="header" href="#must-be-constrained-by-argument-types">Must be constrained by argument types</a></h3> |
| <p>It is important that builtin impls on function item types do not wind up with unconstrained generic parameters as this can lead to unsoundness. This is the same kind of restriction as applies to user written impls, for example the following code results in an error:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait { |
| type Assoc; |
| } |
| |
| impl<'a> Trait for u8 { |
| type Assoc = &'a String; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>The analogous example for builtin impls on function items would be the following:</p> |
| <pre><code class="language-rust ignore">fn foo<'a>() -> &'a String { /* ... */ }</code></pre> |
| <p>If the lifetime parameter <code>'a</code> were to be late bound we would wind up with a builtin impl with an unconstrained lifetime, we can manually write out the desugaring for the function item type and its impls with <code>'a</code> being late bound to demonstrate this:</p> |
| <pre><code class="language-rust ignore">// NOTE: this is just for demonstration, in practice `'a` is early bound |
| struct FooFnItem; |
| |
| impl<'a> Fn<()> for FooFnItem { |
| type Output = &'a String; |
| /* fn call(...) -> ... { ... } */ |
| }</code></pre> |
| <p>In order to avoid such a situation we consider <code>'a</code> to be early bound which causes the lifetime on the impl to be constrained by the self type:</p> |
| <pre><code class="language-rust ignore">struct FooFnItem<'a>(PhantomData<fn() -> &'a String>); |
| |
| impl<'a> Fn<()> for FooFnItem<'a> { |
| type Output = &'a String; |
| /* fn call(...) -> ... { ... } */ |
| }</code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-ty-module-representing-types"><a class="header" href="#the-ty-module-representing-types">The <code>ty</code> module: representing types</a></h1> |
| <p>The <code>ty</code> module defines how the Rust compiler represents types internally. It also defines the |
| <em>typing context</em> (<code>tcx</code> or <code>TyCtxt</code>), which is the central data structure in the compiler.</p> |
| <h2 id="tyty-1"><a class="header" href="#tyty-1"><code>ty::Ty</code></a></h2> |
| <p>When we talk about how rustc represents types, we usually refer to a type called <code>Ty</code> . There are |
| quite a few modules and types for <code>Ty</code> in the compiler (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/index.html">Ty documentation</a>).</p> |
| <p>The specific <code>Ty</code> we are referring to is <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>rustc_middle::ty::Ty</code></a> (and not |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Ty.html"><code>rustc_hir::Ty</code></a>). The distinction is important, so we will discuss it first before going |
| into the details of <code>ty::Ty</code>.</p> |
| <h2 id="rustc_hirty-vs-tyty"><a class="header" href="#rustc_hirty-vs-tyty"><code>rustc_hir::Ty</code> vs <code>ty::Ty</code></a></h2> |
| <p>The HIR in rustc can be thought of as the high-level intermediate representation. It is more or less |
| the AST (see <a href="hir.html">this chapter</a>) as it represents the |
| syntax that the user wrote, and is obtained after parsing and some <em>desugaring</em>. It has a |
| representation of types, but in reality it reflects more of what the user wrote, that is, what they |
| wrote so as to represent that type.</p> |
| <p>In contrast, <code>ty::Ty</code> represents the semantics of a type, that is, the <em>meaning</em> of what the user |
| wrote. For example, <code>rustc_hir::Ty</code> would record the fact that a user used the name <code>u32</code> twice |
| in their program, but the <code>ty::Ty</code> would record the fact that both usages refer to the same type.</p> |
| <p><strong>Example: <code>fn foo(x: u32) → u32 { x }</code></strong></p> |
| <p>In this function, we see that <code>u32</code> appears twice. We know |
| that that is the same type, |
| i.e. the function takes an argument and returns an argument of the same type, |
| but from the point of view of the HIR, |
| there would be two distinct type instances because these |
| are occurring in two different places in the program. |
| That is, they have two different <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html"><code>Span</code>s</a> (locations).</p> |
| <p><strong>Example: <code>fn foo(x: &u32) -> &u32</code></strong></p> |
| <p>In addition, HIR might have information left out. This type |
| <code>&u32</code> is incomplete, since in the full Rust type there is actually a lifetime, but we didn’t need |
| to write those lifetimes. There are also some elision rules that insert information. The result may |
| look like <code>fn foo<'a>(x: &'a u32) -> &'a u32</code>.</p> |
| <p>In the HIR level, these things are not spelled out and you can say the picture is rather incomplete. |
| However, at the <code>ty::Ty</code> level, these details are added and it is complete. Moreover, we will have |
| exactly one <code>ty::Ty</code> for a given type, like <code>u32</code>, and that <code>ty::Ty</code> is used for all <code>u32</code>s in the |
| whole program, not a specific usage, unlike <code>rustc_hir::Ty</code>.</p> |
| <p>Here is a summary:</p> |
| <div class="table-wrapper"><table><thead><tr><th><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Ty.html"><code>rustc_hir::Ty</code></a></th><th><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>ty::Ty</code></a></th></tr></thead><tbody> |
| <tr><td>Describe the <em>syntax</em> of a type: what the user wrote (with some desugaring).</td><td>Describe the <em>semantics</em> of a type: the meaning of what the user wrote.</td></tr> |
| <tr><td>Each <code>rustc_hir::Ty</code> has its own spans corresponding to the appropriate place in the program.</td><td>Doesn’t correspond to a single place in the user’s program.</td></tr> |
| <tr><td><code>rustc_hir::Ty</code> has generics and lifetimes; however, some of those lifetimes are special markers like <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.LifetimeKind.html#variant.Implicit"><code>LifetimeKind::Implicit</code></a>.</td><td><code>ty::Ty</code> has the full type, including generics and lifetimes, even if the user left them out</td></tr> |
| <tr><td><code>fn foo(x: u32) -> u32 { }</code> - Two <code>rustc_hir::Ty</code> representing each usage of <code>u32</code>, each has its own <code>Span</code>s, and <code>rustc_hir::Ty</code> doesn’t tell us that both are the same type</td><td><code>fn foo(x: u32) -> u32 { }</code> - One <code>ty::Ty</code> for all instances of <code>u32</code> throughout the program, and <code>ty::Ty</code> tells us that both usages of <code>u32</code> mean the same type.</td></tr> |
| <tr><td><code>fn foo(x: &u32) -> &u32 { }</code> - Two <code>rustc_hir::Ty</code> again. Lifetimes for the references show up in the <code>rustc_hir::Ty</code>s using a special marker, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.LifetimeKind.html#variant.Implicit"><code>LifetimeKind::Implicit</code></a>.</td><td><code>fn foo(x: &u32) -> &u32 { }</code>- A single <code>ty::Ty</code>. The <code>ty::Ty</code> has the hidden lifetime param.</td></tr> |
| </tbody></table> |
| </div> |
| <p><strong>Order</strong></p> |
| <p>HIR is built directly from the AST, so it happens before any <code>ty::Ty</code> is produced. After |
| HIR is built, some basic type inference and type checking is done. During the type inference, we |
| figure out what the <code>ty::Ty</code> of everything is and we also check if the type of something is |
| ambiguous. The <code>ty::Ty</code> is then used for type checking while making sure everything has the |
| expected type. The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/hir_ty_lowering/index.html"><code>hir_ty_lowering</code> module</a> is where the code responsible for |
| lowering a <code>rustc_hir::Ty</code> to a <code>ty::Ty</code> is located. The main routine used is <code>lower_ty</code>. |
| This occurs during the type-checking phase, but also in other parts of the compiler that want to ask |
| questions like "what argument types does this function expect?"</p> |
| <p><strong>How semantics drive the two instances of <code>Ty</code></strong></p> |
| <p>You can think of HIR as the perspective |
| of the type information that assumes the least. We assume two things are distinct until they are |
| proven to be the same thing. In other words, we know less about them, so we should assume less about |
| them.</p> |
| <p>They are syntactically two strings: <code>"u32"</code> at line N column 20 and <code>"u32"</code> at line N column 35. We |
| don’t know that they are the same yet. So, in the HIR we treat them as if they are different. Later, |
| we determine that they semantically are the same type and that’s the <code>ty::Ty</code> we use.</p> |
| <p>Consider another example: <code>fn foo<T>(x: T) -> u32</code>. Suppose that someone invokes <code>foo::<u32>(0)</code>. |
| This means that <code>T</code> and <code>u32</code> (in this invocation) actually turns out to be the same type, so we |
| would eventually end up with the same <code>ty::Ty</code> in the end, but we have distinct <code>rustc_hir::Ty</code>. |
| (This is a bit over-simplified, though, since during type checking, we would check the function |
| generically and would still have a <code>T</code> distinct from <code>u32</code>. Later, when doing code generation, |
| we would always be handling "monomorphized" (fully substituted) versions of each function, |
| and hence we would know what <code>T</code> represents (and specifically that it is <code>u32</code>).)</p> |
| <p>Here is one more example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>mod a { |
| type X = u32; |
| pub fn foo(x: X) -> u32 { 22 } |
| } |
| mod b { |
| type X = i32; |
| pub fn foo(x: X) -> i32 { x } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Here the type <code>X</code> will vary depending on context, clearly. If you look at the <code>rustc_hir::Ty</code>, |
| you will get back that <code>X</code> is an alias in both cases (though it will be mapped via name resolution |
| to distinct aliases). But if you look at the <code>ty::Ty</code> signature, it will be either <code>fn(u32) -> u32</code> |
| or <code>fn(i32) -> i32</code> (with type aliases fully expanded).</p> |
| <h2 id="tyty-implementation"><a class="header" href="#tyty-implementation"><code>ty::Ty</code> implementation</a></h2> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>rustc_middle::ty::Ty</code></a> is actually a wrapper around |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html"><code>Interned<WithCachedTypeInfo<TyKind>></code></a>. |
| You can ignore <code>Interned</code> in general; you will basically never access it explicitly. |
| We always hide them within <code>Ty</code> and skip over it via <code>Deref</code> impls or methods. |
| <code>TyKind</code> is a big enum |
| with variants to represent many different Rust types |
| (e.g. primitives, references, algebraic data types, generics, lifetimes, etc). |
| <code>WithCachedTypeInfo</code> has a few cached values like <code>flags</code> and <code>outer_exclusive_binder</code>. They |
| are convenient hacks for efficiency and summarize information about the type that we may want to |
| know, but they don’t come into the picture as much here. Finally, <a href="./memory.html"><code>Interned</code></a> allows |
| the <code>ty::Ty</code> to be a thin pointer-like |
| type. This allows us to do cheap comparisons for equality, along with the other |
| benefits of interning.</p> |
| <h2 id="allocating-and-working-with-types"><a class="header" href="#allocating-and-working-with-types">Allocating and working with types</a></h2> |
| <p>To allocate a new type, you can use the various <code>new_*</code> methods defined on |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html"><code>Ty</code></a>. |
| These have names |
| that correspond mostly to the various kinds of types. For example:</p> |
| <pre><code class="language-rust ignore">let array_ty = Ty::new_array_with_const_len(tcx, ty, count);</code></pre> |
| <p>These methods all return a <code>Ty<'tcx></code> – note that the lifetime you get back is the lifetime of the |
| arena that this <code>tcx</code> has access to. Types are always canonicalized and interned (so we never |
| allocate exactly the same type twice).</p> |
| <p>You can also find various common types in the <code>tcx</code> itself by accessing its fields: |
| <code>tcx.types.bool</code>, <code>tcx.types.char</code>, etc. (See <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.CommonTypes.html"><code>CommonTypes</code></a> for more.)</p> |
| <!-- N.B: This section is linked from the type comparison internal lint. --> |
| <h2 id="comparing-types"><a class="header" href="#comparing-types">Comparing types</a></h2> |
| <p>Because types are interned, it is possible to compare them for equality efficiently using <code>==</code> |
| – however, this is almost never what you want to do unless you happen to be hashing and looking |
| for duplicates. This is because often in Rust there are multiple ways to represent the same type, |
| particularly once inference is involved.</p> |
| <p>For example, the type <code>{integer}</code> (<code>ty::Infer(ty::IntVar(..))</code> an integer inference variable, |
| the type of an integer literal like <code>0</code>) and <code>u8</code> (<code>ty::UInt(..)</code>) should often be treated as |
| equal when testing whether they can be assigned to each other (which is a common operation in |
| diagnostics code). <code>==</code> on them will return <code>false</code> though, since they are different types.</p> |
| <p>The simplest way to compare two types correctly requires an inference context (<code>infcx</code>). |
| If you have one, you can use <code>infcx.can_eq(param_env, ty1, ty2)</code> |
| to check whether the types can be made equal. |
| This is typically what you want to check during diagnostics, which is concerned with questions such |
| as whether two types can be assigned to each other, not whether they're represented identically in |
| the compiler's type-checking layer.</p> |
| <p>When working with an inference context, you have to be careful to ensure that potential inference |
| variables inside the types actually belong to that inference context. If you are in a function |
| that has access to an inference context already, this should be the case. Specifically, this is the |
| case during HIR type checking or MIR borrow checking.</p> |
| <p>Another consideration is normalization. Two types may actually be the same, but one is behind an |
| associated type. To compare them correctly, you have to normalize the types first. This is |
| primarily a concern during HIR type checking and with all types from a <code>TyCtxt</code> query |
| (for example from <code>tcx.type_of()</code>).</p> |
| <p>When a <code>FnCtxt</code> or an <code>ObligationCtxt</code> is available during type checking, <code>.normalize(ty)</code> |
| should be used on them to normalize the type. After type checking, diagnostics code can use |
| <code>tcx.normalize_erasing_regions(ty)</code>.</p> |
| <p>There are also cases where using <code>==</code> on <code>Ty</code> is fine. This is for example the case in late lints |
| or after monomorphization, since type checking has been completed, meaning all inference variables |
| are resolved and all regions have been erased. In these cases, if you know that inference variables |
| or normalization won't be a concern, <code>#[allow]</code> or <code>#[expect]</code>ing the lint is recommended.</p> |
| <p>When diagnostics code does not have access to an inference context, it should be threaded through |
| the function calls if one is available in some place (like during type checking).</p> |
| <p>If no inference context is available at all, then one can be created as described in |
| <a href="./type-inference.html#creating-an-inference-context">type-inference</a>. But this is only useful when the involved types (for example, if |
| they came from a query like <code>tcx.type_of()</code>) are actually substituted with fresh |
| inference variables using <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_infer/infer/struct.InferCtxt.html#method.fresh_substs_for_item"><code>fresh_args_for_item</code></a>. This can be used to answer questions |
| like "can <code>Vec<T></code> for any <code>T</code> be unified with <code>Vec<u32></code>?".</p> |
| <h2 id="tytykind-variants"><a class="header" href="#tytykind-variants"><code>ty::TyKind</code> Variants</a></h2> |
| <p>Note: <code>TyKind</code> is <strong>NOT</strong> the functional programming concept of <em>Kind</em>.</p> |
| <p>Whenever working with a <code>Ty</code> in the compiler, it is common to match on the kind of type:</p> |
| <pre><code class="language-rust ignore">fn foo(x: Ty<'tcx>) { |
| match x.kind { |
| ... |
| } |
| }</code></pre> |
| <p>The <code>kind</code> field is of type <code>TyKind<'tcx></code>, which is an enum defining all of the different kinds of |
| types in the compiler.</p> |
| <blockquote> |
| <p>N.B. inspecting the <code>kind</code> field on types during type inference can be risky, as there may be |
| inference variables and other things to consider, or sometimes types are not yet known and will |
| become known later.</p> |
| </blockquote> |
| <p>There are a lot of related types, and we’ll cover them in time (e.g regions/lifetimes, |
| “substitutions”, etc).</p> |
| <p>There are many variants on the <code>TyKind</code> enum, which you can see by looking at its |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html">documentation</a>. Here is a sampling:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Adt"><strong>Algebraic Data Types (ADTs)</strong></a> An <a href="https://en.wikipedia.org/wiki/Algebraic_data_type"><em>algebraic data type</em></a> is a <code>struct</code>, |
| <code>enum</code> or <code>union</code>. Under the hood, <code>struct</code>, <code>enum</code> and <code>union</code> are actually implemented |
| the same way: they are all <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Adt"><code>ty::TyKind::Adt</code></a>. It’s basically a user defined type. |
| We will talk more about these later.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Foreign"><strong>Foreign</strong></a> Corresponds to <code>extern type T</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Str"><strong>Str</strong></a> Is the type str. When the user writes <code>&str</code>, <code>Str</code> is the how we represent the |
| <code>str</code> part of that type.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Slice"><strong>Slice</strong></a> Corresponds to <code>[T]</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Array"><strong>Array</strong></a> Corresponds to <code>[T; n]</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.RawPtr"><strong>RawPtr</strong></a> Corresponds to <code>*mut T</code> or <code>*const T</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Ref"><strong>Ref</strong></a> <code>Ref</code> stands for safe references, <code>&'a mut T</code> or <code>&'a T</code>. <code>Ref</code> has some |
| associated parts, like <code>Ty<'tcx></code> which is the type that the reference references. |
| <code>Region<'tcx></code> is the lifetime or region of the reference and <code>Mutability</code> if the reference |
| is mutable or not.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Param"><strong>Param</strong></a> Represents a type parameter (e.g. the <code>T</code> in <code>Vec<T></code>).</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variant.Error"><strong>Error</strong></a> Represents a type error somewhere so that we can print better diagnostics. We |
| will discuss this more later.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/ty_kind/enum.TyKind.html#variants"><strong>And many more</strong>...</a></li> |
| </ul> |
| <h2 id="import-conventions"><a class="header" href="#import-conventions">Import conventions</a></h2> |
| <p>Although there is no hard and fast rule, the <code>ty</code> module tends to be used like so:</p> |
| <pre><code class="language-rust ignore">use ty::{self, Ty, TyCtxt};</code></pre> |
| <p>In particular, since they are so common, the <code>Ty</code> and <code>TyCtxt</code> types are imported directly. Other |
| types are often referenced with an explicit <code>ty::</code> prefix (e.g. <code>ty::TraitRef<'tcx></code>). But some |
| modules choose to import a larger or smaller set of names explicitly.</p> |
| <h2 id="type-errors"><a class="header" href="#type-errors">Type errors</a></h2> |
| <p>There is a <code>TyKind::Error</code> that is produced when the user makes a type error. The idea is that |
| we would propagate this type and suppress other errors that come up due to it so as not to overwhelm |
| the user with cascading compiler error messages.</p> |
| <p>There is an <strong>important invariant</strong> for <code>TyKind::Error</code>. The compiler should |
| <strong>never</strong> produce <code>Error</code> unless we <strong>know</strong> that an error has already been |
| reported to the user. This is usually |
| because (a) you just reported it right there or (b) you are propagating an existing Error type (in |
| which case the error should've been reported when that error type was produced).</p> |
| <p>It's important to maintain this invariant because the whole point of the <code>Error</code> type is to suppress |
| other errors -- i.e., we don't report them. If we were to produce an <code>Error</code> type without actually |
| emitting an error to the user, then this could cause later errors to be suppressed, and the |
| compilation might inadvertently succeed!</p> |
| <p>Sometimes there is a third case. You believe that an error has been reported, but you believe it |
| would've been reported earlier in the compilation, not locally. In that case, you can create a |
| "delayed bug" with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html#method.delayed_bug"><code>delayed_bug</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html#method.span_delayed_bug"><code>span_delayed_bug</code></a>. This will make a note that you expect |
| compilation to yield an error -- if however compilation should succeed, then it will trigger a |
| compiler bug report.</p> |
| <p>For added safety, it's not actually possible to produce a <code>TyKind::Error</code> value |
| outside of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/index.html"><code>rustc_middle::ty</code></a>; there is a private member of |
| <code>TyKind::Error</code> that prevents it from being constructable elsewhere. Instead, |
| one should use the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html#method.new_error"><code>Ty::new_error</code></a> or |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html#method.new_error_with_message"><code>Ty::new_error_with_message</code></a> methods. These methods either take an <code>ErrorGuaranteed</code> |
| or call <code>span_delayed_bug</code> before returning an interned <code>Ty</code> of kind <code>Error</code>. If you |
| were already planning to use <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.DiagCtxt.html#method.span_delayed_bug"><code>span_delayed_bug</code></a>, then you can just pass the |
| span and message to <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html#method.new_error_with_message"><code>ty_error_with_message</code></a> instead to avoid |
| a redundant delayed bug.</p> |
| <h2 id="tykind-variant-shorthand-syntax"><a class="header" href="#tykind-variant-shorthand-syntax"><code>TyKind</code> variant shorthand syntax</a></h2> |
| <p>When looking at the debug output of <code>Ty</code> or simply talking about different types in the compiler, you may encounter syntax that is not valid rust but is used to concisely represent internal information about types. Below is a quick reference cheat sheet to tell what the various syntax actually means, these should be covered in more depth in later chapters.</p> |
| <ul> |
| <li>Generic parameters: <code>{name}/#{index}</code> e.g. <code>T/#0</code>, where <code>index</code> corresponds to its position in the list of generic parameters</li> |
| <li>Inference variables: <code>?{id}</code> e.g. <code>?x</code>/<code>?0</code>, where <code>id</code> identifies the inference variable</li> |
| <li>Variables from binders: <code>^{binder}_{index}</code> e.g. <code>^0_x</code>/<code>^0_2</code>, where <code>binder</code> and <code>index</code> identify which variable from which binder is being referred to</li> |
| <li>Placeholders: <code>!{id}</code> or <code>!{id}_{universe}</code> e.g. <code>!x</code>/<code>!0</code>/<code>!x_2</code>/<code>!0_2</code>, representing some unique type in the specified universe. The universe is often elided when it is <code>0</code></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="adts-and-generic-arguments"><a class="header" href="#adts-and-generic-arguments">ADTs and Generic Arguments</a></h1> |
| <p>The term <code>ADT</code> stands for "Algebraic data type", in rust this refers to a struct, enum, or union.</p> |
| <h2 id="adts-representation"><a class="header" href="#adts-representation">ADTs Representation</a></h2> |
| <p>Let's consider the example of a type like <code>MyStruct<u32></code>, where <code>MyStruct</code> is defined like so:</p> |
| <pre><code class="language-rust ignore">struct MyStruct<T> { x: u8, y: T }</code></pre> |
| <p>The type <code>MyStruct<u32></code> would be an instance of <code>TyKind::Adt</code>:</p> |
| <pre><code class="language-rust ignore">Adt(&'tcx AdtDef, GenericArgs<'tcx>) |
| // ------------ --------------- |
| // (1) (2) |
| // |
| // (1) represents the `MyStruct` part |
| // (2) represents the `<u32>`, or "substitutions" / generic arguments</code></pre> |
| <p>There are two parts:</p> |
| <ul> |
| <li>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.AdtDef.html"><code>AdtDef</code></a> references the struct/enum/union but without the values for its type |
| parameters. In our example, this is the <code>MyStruct</code> part <em>without</em> the argument <code>u32</code>. |
| (Note that in the HIR, structs, enums and unions are represented differently, but in <code>ty::Ty</code>, |
| they are all represented using <code>TyKind::Adt</code>.)</li> |
| <li>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.GenericArgs.html"><code>GenericArgs</code></a> is a list of values that are to be substituted |
| for the generic parameters. In our example of <code>MyStruct<u32></code>, we would end up with a list like |
| <code>[u32]</code>. We’ll dig more into generics and substitutions in a little bit.</li> |
| </ul> |
| <h3 id="adtdef-and-defid"><a class="header" href="#adtdef-and-defid"><strong><code>AdtDef</code> and <code>DefId</code></strong></a></h3> |
| <p>For every type defined in the source code, there is a unique <code>DefId</code> (see <a href="ty_module/../hir.html#identifiers-in-the-hir">this |
| chapter</a>). This includes ADTs and generics. In the <code>MyStruct<T></code> |
| definition we gave above, there are two <code>DefId</code>s: one for <code>MyStruct</code> and one for <code>T</code>. Notice that |
| the code above does not generate a new <code>DefId</code> for <code>u32</code> because it is not defined in that code (it |
| is only referenced).</p> |
| <p><code>AdtDef</code> is more or less a wrapper around <code>DefId</code> with lots of useful helper methods. There is |
| essentially a one-to-one relationship between <code>AdtDef</code> and <code>DefId</code>. You can get the <code>AdtDef</code> for a |
| <code>DefId</code> with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.adt_def"><code>tcx.adt_def(def_id)</code> query</a>. <code>AdtDef</code>s are all interned, as shown |
| by the <code>'tcx</code> lifetime.</p> |
| <h2 id="question-why-not-substitute-inside-the-adtdef"><a class="header" href="#question-why-not-substitute-inside-the-adtdef">Question: Why not substitute “inside” the <code>AdtDef</code>?</a></h2> |
| <p>Recall that we represent a generic struct with <code>(AdtDef, args)</code>. So why bother with this scheme?</p> |
| <p>Well, the alternate way we could have chosen to represent types would be to always create a new, |
| fully-substituted form of the <code>AdtDef</code> where all the types are already substituted. This seems like |
| less of a hassle. However, the <code>(AdtDef, args)</code> scheme has some advantages over this.</p> |
| <p>First, <code>(AdtDef, args)</code> scheme has an efficiency win:</p> |
| <pre><code class="language-rust ignore">struct MyStruct<T> { |
| ... 100s of fields ... |
| } |
| |
| // Want to do: MyStruct<A> ==> MyStruct<B></code></pre> |
| <p>in an example like this, we can instantiate <code>MyStruct<A></code> as <code>MyStruct<B></code> (and so on) very cheaply, |
| by just replacing the one reference to <code>A</code> with <code>B</code>. But if we eagerly instantiated all the fields, |
| that could be a lot more work because we might have to go through all of the fields in the <code>AdtDef</code> |
| and update all of their types.</p> |
| <p>A bit more deeply, this corresponds to structs in Rust being <a href="https://en.wikipedia.org/wiki/Nominal_type_system"><em>nominal</em> types</a> — which |
| means that they are defined by their <em>name</em> (and that their contents are then indexed from the |
| definition of that name, and not carried along “within” the type itself).</p> |
| <h2 id="the-genericargs-type"><a class="header" href="#the-genericargs-type">The <code>GenericArgs</code> type</a></h2> |
| <p>Given a generic type <code>MyType<A, B, …></code>, we have to store the list of generic arguments for <code>MyType</code>.</p> |
| <p>In rustc this is done using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.GenericArgs.html"><code>GenericArgs</code></a>. <code>GenericArgs</code> is a thin pointer to a slice of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.GenericArg.html"><code>GenericArg</code></a> representing a list of generic arguments for a generic item. For example, given a <code>struct HashMap<K, V></code> with two type parameters, <code>K</code> and <code>V</code>, the <code>GenericArgs</code> used to represent the type <code>HashMap<i32, u32></code> would be represented by <code>&'tcx [tcx.types.i32, tcx.types.u32]</code>.</p> |
| <p><code>GenericArg</code> is conceptually an <code>enum</code> with three variants, one for type arguments, one for const arguments and one for lifetime arguments. |
| In practice that is actually represented by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.GenericArgKind.html"><code>GenericArgKind</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.GenericArg.html"><code>GenericArg</code></a> is a more space efficient version that has a method to |
| turn it into a <code>GenericArgKind</code>.</p> |
| <p>The actual <code>GenericArg</code> struct stores the type, lifetime or const as an interned pointer with the discriminant stored in the lower 2 bits. |
| Unless you are working with the <code>GenericArgs</code> implementation specifically, you should generally not have to deal with <code>GenericArg</code> and instead |
| make use of the safe <a href="ty_module/generic_arguments.html#genericargkind"><code>GenericArgKind</code></a> abstraction obtainable via the <code>GenericArg::unpack()</code> method.</p> |
| <p>In some cases you may have to construct a <code>GenericArg</code>, this can be done via <code>Ty/Const/Region::into()</code> or <code>GenericArgKind::pack</code>.</p> |
| <pre><code class="language-rust ignore">// An example of unpacking and packing a generic argument. |
| fn deal_with_generic_arg<'tcx>(generic_arg: GenericArg<'tcx>) -> GenericArg<'tcx> { |
| // Unpack a raw `GenericArg` to deal with it safely. |
| let new_generic_arg: GenericArgKind<'tcx> = match generic_arg.unpack() { |
| GenericArgKind::Type(ty) => { /* ... */ } |
| GenericArgKind::Lifetime(lt) => { /* ... */ } |
| GenericArgKind::Const(ct) => { /* ... */ } |
| }; |
| // Pack the `GenericArgKind` to store it in a generic args list. |
| new_generic_arg.pack() |
| }</code></pre> |
| <p>So pulling it all together:</p> |
| <pre><code class="language-rust ignore">struct MyStruct<T>(T); |
| type Foo = MyStruct<u32></code></pre> |
| <p>For the <code>MyStruct<U></code> written in the <code>Foo</code> type alias, we would represent it in the following way:</p> |
| <ul> |
| <li>There would be an <code>AdtDef</code> (and corresponding <code>DefId</code>) for <code>MyStruct</code>.</li> |
| <li>There would be a <code>GenericArgs</code> containing the list <code>[GenericArgKind::Type(Ty(u32))]</code></li> |
| <li>And finally a <code>TyKind::Adt</code> with the <code>AdtDef</code> and <code>GenericArgs</code> listed above.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="parameter-tyconstregions"><a class="header" href="#parameter-tyconstregions">Parameter <code>Ty</code>/<code>Const</code>/<code>Region</code>s</a></h1> |
| <p>When inside of generic items, types can be written that use in scope generic parameters, for example <code>fn foo<'a, T>(_: &'a Vec<T>)</code>. In this specific case |
| the <code>&'a Vec<T></code> type would be represented internally as:</p> |
| <pre><code>TyKind::Ref( |
| RegionKind::LateParam(DefId(foo), DefId(foo::'a), "'a"), |
| TyKind::Adt(Vec, &[TyKind::Param("T", 0)]) |
| ) |
| </code></pre> |
| <p>There are three separate ways we represent usages of generic parameters:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TyKind.html#variant.Param"><code>TyKind::Param</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.ConstKind.html#variant.Param"><code>ConstKind::Param</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.RegionKind.html#variant.ReEarlyParam"><code>RegionKind::EarlyParam</code></a> for early bound generic parameters (note: all type and const parameters are considered early bound, see the <a href="ty_module/../early_late_parameters.html">chapter on early vs late bound parameters</a> for more information)</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TyKind.html#variant.Bound"><code>TyKind::Bound</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.ConstKind.html#variant.Bound"><code>ConstKind::Bound</code></a>/<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.RegionKind.html#variant.ReBound"><code>RegionKind::Bound</code></a> for references to parameters introduced via higher ranked bounds or higher ranked types i.e. <code>for<'a> fn(&'a u32)</code> or <code>for<'a> T: Trait<'a></code>. This is discussed in the <a href="ty_module/./binders.html">chapter on <code>Binder</code>s</a>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.RegionKind.html#variant.ReLateParam"><code>RegionKind::LateParam</code></a> for late bound lifetime parameters, <code>LateParam</code> is discussed in the <a href="ty_module/./instantiating_binders.html">chapter on instantiating <code>Binder</code>s</a>.</li> |
| </ul> |
| <p>This chapter only covers <code>TyKind::Param</code> <code>ConstKind::Param</code> and <code>RegionKind::EarlyParam</code>.</p> |
| <h2 id="tyconst-parameters"><a class="header" href="#tyconst-parameters">Ty/Const Parameters</a></h2> |
| <p>As <code>TyKind::Param</code> and <code>ConstKind::Param</code> are implemented identically this section only refers to <code>TyKind::Param</code> for simplicity. |
| However you should keep in mind that everything here also is true of <code>ConstKind::Param</code></p> |
| <p>Each <code>TyKind::Param</code> contains two things: the name of the parameter and an index.</p> |
| <p>See the following concrete example of a usage of <code>TyKind::Param</code>:</p> |
| <pre><code class="language-rust ignore">struct Foo<T>(Vec<T>);</code></pre> |
| <p>The <code>Vec<T></code> type is represented as <code>TyKind::Adt(Vec, &[GenericArgKind::Type(Param("T", 0))])</code>.</p> |
| <p>The name is somewhat self explanatory, it's the name of the type parameter. The index of the type parameter is an integer indicating |
| its order in the list of generic parameters in scope (note: this includes parameters defined on items on outer scopes than the item the parameter is defined on). Consider the following examples:</p> |
| <pre><code class="language-rust ignore">struct Foo<A, B> { |
| // A would have index 0 |
| // B would have index 1 |
| |
| .. // some fields |
| } |
| impl<X, Y> Foo<X, Y> { |
| fn method<Z>() { |
| // inside here, X, Y and Z are all in scope |
| // X has index 0 |
| // Y has index 1 |
| // Z has index 2 |
| } |
| }</code></pre> |
| <p>Concretely given the <code>ty::Generics</code> for the item the parameter is defined on, if the index is <code>2</code> then starting from the root <code>parent</code>, it will be the third parameter to be introduced. For example in the above example, <code>Z</code> has index <code>2</code> and is the third generic parameter to be introduced, starting from the <code>impl</code> block.</p> |
| <p>The index fully defines the <code>Ty</code> and is the only part of <code>TyKind::Param</code> that matters for reasoning about the code we are compiling.</p> |
| <p>Generally we do not care what the name is and only use the index. The name is included for diagnostics and debug logs as otherwise it would be |
| incredibly difficult to understand the output, i.e. <code>Vec<Param(0)>: Sized</code> vs <code>Vec<T>: Sized</code>. In debug output, parameter types are |
| often printed out as <code>{name}/#{index}</code>, for example in the function <code>foo</code> if we were to debug print <code>Vec<T></code> it would be written as <code>Vec<T/#0></code>.</p> |
| <p>An alternative representation would be to only have the name, however using an index is more efficient as it means we can index into <code>GenericArgs</code> when instantiating generic parameters with some arguments. We would otherwise have to store <code>GenericArgs</code> as a <code>HashMap<Symbol, GenericArg></code> and do a hashmap lookup everytime we used a generic item.</p> |
| <p>In theory an index would also allow for having multiple distinct parameters that use the same name, e.g. |
| <code>impl<A> Foo<A> { fn bar<A>() { .. } }</code>. |
| The rules against shadowing make this difficult but those language rules could change in the future.</p> |
| <h3 id="lifetime-parameters"><a class="header" href="#lifetime-parameters">Lifetime parameters</a></h3> |
| <p>In contrast to <code>Ty</code>/<code>Const</code>'s <code>Param</code> singular <code>Param</code> variant, lifetimes have two variants for representing region parameters: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.RegionKind.html#variant.ReEarlyParam"><code>RegionKind::EarlyParam</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.RegionKind.html#variant.ReLateParam"><code>RegionKind::LateParam</code></a>. The reason for this is due to function's distinguishing between <a href="ty_module/../early_late_parameters.html">early and late bound parameters</a> which is discussed in an earlier chapter (see link).</p> |
| <p><code>RegionKind::EarlyParam</code> is structured identically to <code>Ty/Const</code>'s <code>Param</code> variant, it is simply a <code>u32</code> index and a <code>Symbol</code>. For lifetime parameters defined on non-function items we always use <code>ReEarlyParam</code>. For functions we use <code>ReEarlyParam</code> for any early bound parameters and <code>ReLateParam</code> for any late bound parameters. Note that just like <code>Ty</code> and <code>Const</code> params we often debug format them as <code>'SYMBOL/#INDEX</code>, see for example:</p> |
| <pre><code class="language-rust ignore">// This function would have its signature represented as: |
| // |
| // ``` |
| // fn( |
| // T/#2, |
| // Ref('a/#0, Ref(ReLateParam(...), u32)) |
| // ) -> Ref(ReLateParam(...), u32) |
| // ``` |
| fn foo<'a, 'b, T: 'a>(one: T, two: &'a &'b u32) -> &'b u32 { |
| ... |
| }</code></pre> |
| <p><code>RegionKind::LateParam</code> is discussed more in the chapter on <a href="ty_module/./instantiating_binders.html">instantiating binders</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><!-- date-check: may 2024 --> |
| <h1 id="typefoldable-and-typefolder"><a class="header" href="#typefoldable-and-typefolder"><code>TypeFoldable</code> and <code>TypeFolder</code></a></h1> |
| <p>In <a href="ty_module/instantiating_binders.html">a previous chapter</a>, we discussed instantiating binders. |
| This involves looking at everything inside of a <code>Early(Binder)</code> |
| to find any usages of the bound vars in order to replace them. |
| Binders can wrap an arbitrary Rust type <code>T</code>, not just a <code>Ty</code>. |
| So, how do we implement the <code>instantiate</code> methods on the <code>Early/Binder</code> types?</p> |
| <p>The answer is a couple of traits: |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait.TypeFoldable.html"><code>TypeFoldable</code></a> |
| and |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait.TypeFolder.html"><code>TypeFolder</code></a>.</p> |
| <ul> |
| <li><code>TypeFoldable</code> is implemented by types that embed type information. It allows you to recursively |
| process the contents of the <code>TypeFoldable</code> and do stuff to them.</li> |
| <li><code>TypeFolder</code> defines what you want to do with the types you encounter while processing the |
| <code>TypeFoldable</code>.</li> |
| </ul> |
| <p>For example, the <code>TypeFolder</code> trait has a method <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait.TypeFolder.html#method.fold_ty"><code>fold_ty</code></a> |
| that takes a type as input and returns a new type as a result. |
| <code>TypeFoldable</code> invokes the <code>TypeFolder</code> <code>fold_foo</code> methods on itself, |
| giving the <code>TypeFolder</code> access to its contents (the types, regions, etc that are contained within).</p> |
| <p>You can think of it with this analogy to the iterator combinators we have come to love in Rust:</p> |
| <pre><code class="language-rust ignore">vec.iter().map(|e1| foo(e2)).collect() |
| // ^^^^^^^^^^^^ analogous to `TypeFolder` |
| // ^^^ analogous to `TypeFoldable`</code></pre> |
| <p>So to reiterate:</p> |
| <ul> |
| <li><code>TypeFolder</code> is a trait that defines a “map” operation.</li> |
| <li><code>TypeFoldable</code> is a trait that is implemented by things that embed types.</li> |
| </ul> |
| <p>In the case of <code>subst</code>, we can see that it is implemented as a <code>TypeFolder</code>: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/binder/struct.ArgFolder.html"><code>ArgFolder</code></a>. |
| Looking at its implementation, we see where the actual substitutions are happening.</p> |
| <p>However, you might also notice that the implementation calls this <code>super_fold_with</code> method. What is |
| that? It is a method of <code>TypeFoldable</code>. Consider the following <code>TypeFoldable</code> type <code>MyFoldable</code>:</p> |
| <pre><code class="language-rust ignore">struct MyFoldable<'tcx> { |
| def_id: DefId, |
| ty: Ty<'tcx>, |
| }</code></pre> |
| <p>The <code>TypeFolder</code> can call <code>super_fold_with</code> on <code>MyFoldable</code> if it just wants to replace some of the |
| fields of <code>MyFoldable</code> with new values. If it instead wants to replace the whole <code>MyFoldable</code> with a |
| different one, it would call <code>fold_with</code> instead (a different method on <code>TypeFoldable</code>).</p> |
| <p>In almost all cases, we don’t want to replace the whole struct; we only want to replace <code>ty::Ty</code>s in |
| the struct, so usually we call <code>super_fold_with</code>. A typical implementation that <code>MyFoldable</code> could |
| have might do something like this:</p> |
| <pre><code class="language-rust ignore">my_foldable: MyFoldable<'tcx> |
| my_foldable.subst(..., subst) |
| |
| impl TypeFoldable for MyFoldable { |
| fn super_fold_with(&self, folder: &mut impl TypeFolder<'tcx>) -> MyFoldable { |
| MyFoldable { |
| def_id: self.def_id.fold_with(folder), |
| ty: self.ty.fold_with(folder), |
| } |
| } |
| |
| fn super_visit_with(..) { } |
| }</code></pre> |
| <p>Notice that here, we implement <code>super_fold_with</code> to go over the fields of <code>MyFoldable</code> and call |
| <code>fold_with</code> on <em>them</em>. That is, a folder may replace <code>def_id</code> and <code>ty</code>, but not the whole |
| <code>MyFoldable</code> struct.</p> |
| <p>Here is another example to put things together: suppose we have a type like <code>Vec<Vec<X>></code>. The |
| <code>ty::Ty</code> would look like: <code>Adt(Vec, &[Adt(Vec, &[Param(X)])])</code>. If we want to do <code>subst(X => u32)</code>, |
| then we would first look at the overall type. We would see that there are no substitutions to be |
| made at the outer level, so we would descend one level and look at <code>Adt(Vec, &[Param(X)])</code>. There |
| are still no substitutions to be made here, so we would descend again. Now we are looking at |
| <code>Param(X)</code>, which can be substituted, so we replace it with <code>u32</code>. We can’t descend any more, so we |
| are done, and the overall result is <code>Adt(Vec, &[Adt(Vec, &[u32])])</code>.</p> |
| <p>One last thing to mention: often when folding over a <code>TypeFoldable</code>, we don’t want to change most |
| things. We only want to do something when we reach a type. That means there may be a lot of |
| <code>TypeFoldable</code> types whose implementations basically just forward to their fields’ <code>TypeFoldable</code> |
| implementations. Such implementations of <code>TypeFoldable</code> tend to be pretty tedious to write by hand. |
| For this reason, there is a <code>derive</code> macro that allows you to <code>#![derive(TypeFoldable)]</code>. It is |
| defined <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_macros/src/type_foldable.rs">here</a>.</p> |
| <p><strong><code>subst</code></strong> In the case of substitutions the <a href="https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L440-L451">actual folder</a> |
| is going to be doing the indexing we’ve already mentioned. |
| There we define a <code>Folder</code> and call <code>fold_with</code> on the <code>TypeFoldable</code> to process yourself. |
| Then <a href="https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L512-L536">fold_ty</a> the method that process each type it looks for a <code>ty::Param</code> and for those |
| it replaces it for something from the list of substitutions, otherwise recursively process the type. |
| To replace it, calls <a href="https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L552-L587">ty_for_param</a> |
| and all that does is index into the list of substitutions with the index of the <code>Param</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="aliases-and-normalization"><a class="header" href="#aliases-and-normalization">Aliases and Normalization</a></h1> |
| <h2 id="aliases"><a class="header" href="#aliases">Aliases</a></h2> |
| <p>In Rust there are a number of types that are considered equal to some "underlying" type, for example inherent associated types, trait associated types, free type aliases (<code>type Foo = u32</code>), and opaque types (<code>-> impl RPIT</code>). We consider such types to be "aliases", alias types are represented by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/enum.TyKind.html#variant.Alias"><code>TyKind::Alias</code></a> variant, with the kind of alias tracked by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/enum.AliasTyKind.html"><code>AliasTyKind</code></a> enum.</p> |
| <p>Normalization is the process of taking these alias types and replacing them with the underlying type that they are equal to. For example given some type alias <code>type Foo = u32</code>, normalizing <code>Foo</code> would give <code>u32</code>.</p> |
| <p>The concept of an alias is not unique to <em>types</em> and the concept also applies to constants/const generics. However, right now in the compiler we don't really treat const aliases as a "first class concept" so this chapter mostly discusses things in the context of types (even though the concepts transfer just fine).</p> |
| <h3 id="rigid-ambiguous-and-unnormalized-aliases"><a class="header" href="#rigid-ambiguous-and-unnormalized-aliases">Rigid, Ambiguous and Unnormalized Aliases</a></h3> |
| <p>Aliases can either be "rigid", "ambiguous", or simply unnormalized.</p> |
| <p>We consider types to be rigid if their "shape" isn't going to change, for example <code>Box</code> is rigid as no amount of normalization can turn a <code>Box</code> into a <code>u32</code>, whereas <code><vec::IntoIter<u32> as Iterator>::Item</code> is not rigid as it can be normalized to <code>u32</code>.</p> |
| <p>Aliases are rigid when we will never be able to normalize them further. A concrete example of a <em>rigid</em> alias would be <code><T as Iterator>::Item</code> in an environment where there is no <code>T: Iterator<Item = ...></code> bound, only a <code>T: Iterator</code> bound:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T: Iterator>() { |
| // This alias is *rigid* |
| let _: <T as Iterator>::Item; |
| } |
| |
| fn bar<T: Iterator<Item = u32>>() { |
| // This alias is *not* rigid as it can be normalized to `u32` |
| let _: <T as Iterator>::Item; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>When an alias can't yet be normalized but may wind up normalizable in the <a href="./typing_parameter_envs.html">current environment</a>, we consider it to be an "ambiguous" alias. This can occur when an alias contains inference variables which prevent being able to determine how the trait is implemented:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T: Iterator, U: Iterator>() { |
| // This alias is considered to be "ambiguous" |
| let _: <_ as Iterator>::Item; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>The reason we call them "ambiguous" aliases is because its <em>ambiguous</em> whether this is a rigid alias or not.</p> |
| <p>The source of the <code>_: Iterator</code> trait impl is <em>ambiguous</em> (i.e. unknown), it could be some <code>impl Iterator for u32</code> or it could be some <code>T: Iterator</code> trait bound, we don't know yet. Depending on why <code>_: Iterator</code> holds the alias could be an unnormalized alias or it could be a rigid alias; it's <em>ambiguous</em> what kind of alias this is.</p> |
| <p>Finally, an alias can just be unnormalized, <code><Vec<u32> as IntoIterator>::Iter</code> is an unnormalized alias as it can already be normalized to <code>std::vec::IntoIter<u32></code>, it just hasn't been done yet.</p> |
| <hr /> |
| <p>It is worth noting that Free and Inherent aliases cannot be rigid or ambiguous as naming them also implies having resolved the definition of the alias, which specifies the underlying type of the alias.</p> |
| <h3 id="diverging-aliases"><a class="header" href="#diverging-aliases">Diverging Aliases</a></h3> |
| <p>An alias is considered to "diverge" if its definition does not specify an underlying non-alias type to normalize to. A concrete example of diverging aliases:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Diverges = Diverges; |
| |
| trait Trait { |
| type DivergingAssoc; |
| } |
| impl Trait for () { |
| type DivergingAssoc = <() as Trait>::DivergingAssoc; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example both <code>Diverges</code> and <code>DivergingAssoc</code> are "trivial" cases of diverging type aliases where they have been defined as being equal to themselves. There is no underlying type that <code>Diverges</code> can ever be normalized to.</p> |
| <p>We generally try to error when diverging aliases are defined, but this is entirely a "best effort" check. In the previous example the definitions are "simple enough" to be detected and so errors are emitted. However, in more complex cases, or cases where only some instantiations of generic parameters would result in a diverging alias, we don't emit an error:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait { |
| type DivergingAssoc<U: Trait>; |
| } |
| impl<T: ?Sized> Trait for T { |
| // This alias always diverges but we don't emit an error because |
| // the compiler can't "see" that. |
| type DivergingAssoc<U: Trait> = <U as Trait>::DivergingAssoc<U>; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Ultimately this means that we have no guarantee that aliases in the type system are non-diverging. As aliases may only diverge for some specific generic arguments, it also means that we only know whether an alias diverges once it is fully concrete. This means that codegen/const-evaluation also has to handle diverging aliases:</p> |
| <pre><pre class="playground"><code class="language-rust">trait Trait { |
| type Diverges<U: Trait>; |
| } |
| impl<T: ?Sized> Trait for T { |
| type Diverges<U: Trait> = <U as Trait>::Diverges<U>; |
| } |
| |
| fn foo<T: Trait>() { |
| let a: T::Diverges<T>; |
| } |
| |
| fn main() { |
| foo::<()>(); |
| }</code></pre></pre> |
| <p>In this example we only encounter an error from the diverging alias during codegen of <code>foo::<()></code>, if the call to <code>foo</code> is removed then no compilation error will be emitted.</p> |
| <h3 id="opaque-types"><a class="header" href="#opaque-types">Opaque Types</a></h3> |
| <p>Opaque types are a relatively special kind of alias, and are covered in their own chapter: <a href="./opaque-types-type-alias-impl-trait.html">Opaque types</a>.</p> |
| <h3 id="const-aliases"><a class="header" href="#const-aliases">Const Aliases</a></h3> |
| <p>Unlike type aliases, const aliases are not represented directly in the type system, instead const aliases are always an anonymous body containing a path expression to a const item. This means that the only "const alias" in the type system is an anonymous unevaluated const body.</p> |
| <p>As such there is no <code>ConstKind::Alias(AliasCtKind::Projection/Inherent/Free, _)</code>, instead we only have <code>ConstKind::Unevaluated</code> which is used for representing anonymous constants.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<const N: usize>() {} |
| |
| const FREE_CONST: usize = 1 + 1; |
| |
| fn bar() { |
| foo::<{ FREE_CONST }>(); |
| // The const arg is represented with some anonymous constant: |
| // ```pseudo-rust |
| // const ANON: usize = FREE_CONST; |
| // foo::<ConstKind::Unevaluated(DefId(ANON), [])>(); |
| // ``` |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This is likely to change as const generics functionality is improved, for example <code>feature(associated_const_equality)</code> and <code>feature(min_generic_const_args)</code> both require handling const aliases similarly to types (without an anonymous constant wrapping all const args).</p> |
| <h2 id="what-is-normalization"><a class="header" href="#what-is-normalization">What is Normalization</a></h2> |
| <h3 id="structural-vs-deep-normalization"><a class="header" href="#structural-vs-deep-normalization">Structural vs Deep normalization</a></h3> |
| <p>There are two forms of normalization, structural (sometimes called <em>shallow</em>) and deep. Structural normalization should be thought of as only normalizing the "outermost" part of a type. On the other hand deep normalization will normalize <em>all</em> aliases in a type.</p> |
| <p>In practice structural normalization can result in more than just the outer layer of the type being normalized, but this behaviour should not be relied upon. Unnormalizable non-rigid aliases making use of bound variables (<code>for<'a></code>) cannot be normalized by either kind of normalization.</p> |
| <p>As an example: conceptually, structurally normalizing the type <code>Vec<<u8 as Identity>::Assoc></code> would be a no-op, whereas deeply normalizing would give <code>Vec<u8></code>. In practice even structural normalization would give <code>Vec<u8></code>, though, again, this should not be relied upon.</p> |
| <p>Changing the alias to use bound variables will result in different behaviour; <code>Vec<for<'a> fn(<&'a u8 as Identity>::Assoc)></code> would result in no change when structurally normalized, but would result in <code>Vec<for<'a> fn(&'a u8)></code> when deeply normalized.</p> |
| <h3 id="core-normalization-logic"><a class="header" href="#core-normalization-logic">Core normalization logic</a></h3> |
| <p>Structurally normalizing aliases is a little bit more nuanced than replacing the alias with whatever it is defined as being equal to in its definition; the result of normalizing an alias should either be a rigid type or an inference variable (which will later be inferred to a rigid type). To accomplish this we do two things:</p> |
| <p>First, when normalizing an ambiguous alias it is normalized to an inference variable instead of leaving it as-is, this has two main effects:</p> |
| <ul> |
| <li>Even though an inference variable is not a rigid type, it will always wind up inferred <em>to</em> a rigid type so we ensure that the result of normalization will not need to be normalized again</li> |
| <li>Inference variables are used in all cases where a type is non-rigid, allowing the rest of the compiler to not have to deal with <em>both</em> ambiguous aliases <em>and</em> inference variables</li> |
| </ul> |
| <p>Secondly, instead of having normalization directly return the type specified in the definition of the alias, we normalize the type first before returning it<sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup>. We do this so that normalization is idempotent/callers do not need to run it in a loop.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(lazy_type_alias)] |
| |
| <span class="boring">fn main() { |
| </span>type Foo<T: Iterator> = Bar<T>; |
| type Bar<T: Iterator> = <T as Iterator>::Item; |
| |
| fn foo() { |
| let a_: Foo<_>; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example:</p> |
| <ul> |
| <li>Normalizing <code>Foo<?x></code> would result in <code>Bar<?x></code>, except we want to normalize aliases in the type <code>Foo</code> is defined as equal to</li> |
| <li>Normalizing <code>Bar<?x></code> would result in <code><?x as Iterator>::Item</code>, except, again, we want to normalize aliases in the type <code>Bar</code> is defined as equal to</li> |
| <li>Normalizing <code><?x as Iterator>::Item</code> results in some new inference variable <code>?y</code>, as <code><?x as Iterator>::Item</code> is an ambiguous alias</li> |
| <li>The final result is that normalizing <code>Foo<?x></code> results in <code>?y</code></li> |
| </ul> |
| <h2 id="how-to-normalize"><a class="header" href="#how-to-normalize">How to normalize</a></h2> |
| <p>When interfacing with the type system it will often be the case that it's necessary to request a type be normalized. There are a number of different entry points to the underlying normalization logic and each entry point should only be used in specific parts of the compiler.</p> |
| <!-- date-check: May 2025 --> |
| <p>An additional complication is that the compiler is currently undergoing a transition from the old trait solver to the new trait solver. |
| As part of this transition our approach to normalization in the compiler has changed somewhat significantly, resulting in some normalization entry points being "old solver only" slated for removal in the long-term once the new solver has stabilized. |
| The transition can be tracked via the <a href="https://github.com/rust-lang/rust/labels/WG-trait-system-refactor">WG-trait-system-refactor</a> label in Github.</p> |
| <p>Here is a rough overview of the different entry points to normalization in the compiler:</p> |
| <ul> |
| <li><code>infcx.at.structurally_normalize</code></li> |
| <li><code>infcx.at.(deeply_)?normalize</code></li> |
| <li><code>infcx.query_normalize</code></li> |
| <li><code>tcx.normalize_erasing_regions</code></li> |
| <li><code>traits::normalize_with_depth(_to)</code></li> |
| <li><code>EvalCtxt::structurally_normalize</code></li> |
| </ul> |
| <h3 id="outside-of-the-trait-solver"><a class="header" href="#outside-of-the-trait-solver">Outside of the trait solver</a></h3> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/struct.InferCtxt.html"><code>InferCtxt</code></a> type exposes the "main" ways to normalize during analysis: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/at/struct.At.html#method.normalize"><code>normalize</code></a>, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/normalize/trait.NormalizeExt.html#tymethod.deeply_normalize"><code>deeply_normalize</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/trait.StructurallyNormalizeExt.html#tymethod.structurally_normalize_ty"><code>structurally_normalize</code></a>. These functions are often wrapped and re-exposed on various <code>InferCtxt</code> wrapper types, such as <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html"><code>FnCtxt</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/struct.ObligationCtxt.html"><code>ObligationCtxt</code></a> with minor API tweaks to handle some arguments or parts of the return type automatically.</p> |
| <h4 id="structural-inferctxt-normalization"><a class="header" href="#structural-inferctxt-normalization">Structural <code>InferCtxt</code> normalization</a></h4> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/trait.StructurallyNormalizeExt.html#tymethod.structurally_normalize_ty"><code>infcx.at.structurally_normalize</code></a> exposes structural normalization that is able to handle inference variables and regions. It should generally be used whenever inspecting the kind of a type.</p> |
| <p>Inside of HIR Typeck there is a related method of normalization- <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#method.structurally_resolve_type"><code>fcx.structurally_resolve</code></a>, which will error if the type being resolved is an unresolved inference variable. When the new solver is enabled it will also attempt to structurally normalize the type.</p> |
| <p>Due to this there is a pattern in HIR typeck where a type is first normalized via <code>normalize</code> (only normalizing in the old solver), and then <code>structurally_resolve</code>'d (only normalizing in the new solver). This pattern should be preferred over calling <code>structurally_normalize</code> during HIR typeck as <code>structurally_resolve</code> will attempt to make inference progress by evaluating goals whereas <code>structurally_normalize</code> does not.</p> |
| <h4 id="deep-inferctxt-normalization"><a class="header" href="#deep-inferctxt-normalization">Deep <code>InferCtxt</code> normalization</a></h4> |
| <h5 id="infcxatdeeply_normalize"><a class="header" href="#infcxatdeeply_normalize"><code>infcx.at.(deeply_)?normalize</code></a></h5> |
| <p>There are two ways to deeply normalize with an <code>InferCtxt</code>, <code>normalize</code> and <code>deeply_normalize</code>. The reason for this is that <code>normalize</code> is a "legacy" normalization entry point used only by the old solver, whereas <code>deeply_normalize</code> is intended to be the long term way to deeply normalize. Both of these methods can handle regions.</p> |
| <p>When the new solver is stabilized the <code>infcx.at.normalize</code> function will be removed and everything will have been migrated to the new deep or structural normalization methods. For this reason the <code>normalize</code> function is a no-op under the new solver, making it suitable only when the old solver needs normalization but the new solver does not.</p> |
| <p>Using <code>deeply_normalize</code> will result in errors being emitted when encountering ambiguous aliases<sup class="footnote-reference" id="fr-2-1"><a href="#footnote-2">2</a></sup> as it is not possible to support normalizing <em>all</em> ambiguous aliases to inference variables<sup class="footnote-reference" id="fr-3-1"><a href="#footnote-3">3</a></sup>. <code>deeply_normalize</code> should generally only be used in cases where we do not expect to encounter ambiguous aliases, for example when working with types from item signatures.</p> |
| <h5 id="infcxquery_normalize"><a class="header" href="#infcxquery_normalize"><code>infcx.query_normalize</code></a></h5> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/at/struct.At.html#method.query_normalize"><code>infcx.query_normalize</code></a> is very rarely used, it has almost all the same restrictions as <code>normalize_erasing_regions</code> (cannot handle inference variables, no diagnostics support) with the main difference being that it retains lifetime information. For this reason <code>normalize_erasing_regions</code> is the better choice in almost all circumstances as it is more efficient due to caching lifetime-erased queries.</p> |
| <p>In practice <code>query_normalize</code> is used for normalization in the borrow checker, and elsewhere as a performance optimization over <code>infcx.normalize</code>. Once the new solver is stabilized it is expected that <code>query_normalize</code> can be removed from the compiler as the new solvers normalization implementation should be performant enough for it to not be a performance regression.</p> |
| <h5 id="tcxnormalize_erasing_regions"><a class="header" href="#tcxnormalize_erasing_regions"><code>tcx.normalize_erasing_regions</code></a></h5> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.normalize_erasing_regions"><code>normalize_erasing_regions</code></a> is generally used by parts of the compiler that are not doing type system analysis. This normalization entry point does not handle inference variables, lifetimes, or any diagnostics. Lints and codegen make heavy use of this entry point as they typically are working with fully inferred aliases that can be assumed to be well formed (or at least, are not responsible for erroring on).</p> |
| <h3 id="inside-of-the-trait-solver"><a class="header" href="#inside-of-the-trait-solver">Inside of the trait solver</a></h3> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/normalize/fn.normalize_with_depth.html"><code>traits::normalize_with_depth(_to)</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/struct.EvalCtxt.html#method.structurally_normalize_term"><code>EvalCtxt::structurally_normalize</code></a> are only used by the internals of the trait solvers (old and new respectively). It is effectively a raw entry point to the internals of how normalization is implemented by each trait solver. Other normalization entry points cannot be used from within the internals of trait solving as it wouldn't handle goal cycles and recursion depth correctly.</p> |
| <h2 id="whenwhere-to-normalize-old-vs-new-solver"><a class="header" href="#whenwhere-to-normalize-old-vs-new-solver">When/Where to normalize (Old vs New solver)</a></h2> |
| <p>One of the big changes between the old and new solver is our approach to when we expect aliases to be normalized.</p> |
| <h3 id="old-solver"><a class="header" href="#old-solver">Old solver</a></h3> |
| <p>All types are expected to be normalized as soon as possible, so that all types encountered in the type system are either rigid or an inference variable (which will later be inferred to a rigid term).</p> |
| <p>As a concrete example: equality of aliases is implemented by assuming they're rigid and recursively equating the generic arguments of the alias.</p> |
| <h3 id="new-solver"><a class="header" href="#new-solver">New solver</a></h3> |
| <p>It's expected that all types potentially contain ambiguous or unnormalized aliases. Whenever an operation is performed that requires aliases to be normalized, it's the responsibility of that logic to normalize the alias (this means that matching on <code>ty.kind()</code> pretty much always has to structurally normalize first).</p> |
| <p>As a concrete example: equality of aliases is implemented by a custom goal kind (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.PredicateKind.html#variant.AliasRelate"><code>PredicateKind::AliasRelate</code></a>) so that it can handle normalization of the aliases itself instead of assuming all alias types being equated are rigid.</p> |
| <p>Despite this approach we still deeply normalize during <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/writeback/index.html">writeback</a> for performance/simplicity, so that types in the MIR can still be assumed to have been deeply normalized.</p> |
| <hr /> |
| <p>There were a few main issues with the old solver's approach to normalization that motivated changing things in the new solver:</p> |
| <h3 id="missing-normalization-calls"><a class="header" href="#missing-normalization-calls">Missing normalization calls</a></h3> |
| <p>It was a frequent occurrence that normalization calls would be missing, resulting in passing unnormalized types to APIs expecting everything to already be normalized. Treating ambiguous or unnormalized aliases as rigid would result in all sorts of weird errors from aliases not being considered equal to one another, or surprising inference guidance from equating unnormalized aliases' generic arguments.</p> |
| <h3 id="normalizing-parameter-environments"><a class="header" href="#normalizing-parameter-environments">Normalizing parameter environments</a></h3> |
| <p>Another problem was that it was not possible to normalize <code>ParamEnv</code>s correctly in the old solver as normalization itself would expect a normalized <code>ParamEnv</code> in order to give correct results. See the chapter on <code>ParamEnv</code>s for more information: <a href="./typing_parameter_envs.html#normalizing-all-bounds"><code>Typing/ParamEnv</code>s: Normalizing all bounds</a></p> |
| <h3 id="unnormalizable-non-rigid-aliases-in-higher-ranked-types"><a class="header" href="#unnormalizable-non-rigid-aliases-in-higher-ranked-types">Unnormalizable non-rigid aliases in higher ranked types</a></h3> |
| <p>Given a type such as <code>for<'a> fn(<?x as Trait<'a>::Assoc>)</code>, it is not possible to correctly handle this with the old solver's approach to normalization.</p> |
| <p>If we were to normalize it to <code>for<'a> fn(?y)</code> and register a goal to normalize <code>for<'a> <?x as Trait<'a>>::Assoc -> ?y</code>, this would result in errors in cases where <code><?x as Trait<'a>>::Assoc</code> normalized to <code>&'a u32</code>. The inference variable <code>?y</code> would be in a lower <a href="borrow_check/region_inference/placeholders_and_universes.html#what-is-a-universe">universe</a> than the placeholders made when instantiating the <code>for<'a></code> binder.</p> |
| <p>Leaving the alias unnormalized would also be wrong as the old solver expects all aliases to be rigid. This was a soundness bug before the new solver was stabilized in coherence: <a href="https://github.com/rust-lang/rust/issues/102048">relating projection substs is unsound during coherence</a>.</p> |
| <p>Ultimately this means that it is not always possible to ensure all aliases inside of a value are rigid.</p> |
| <h2 id="handling-uses-of-diverging-aliases"><a class="header" href="#handling-uses-of-diverging-aliases">Handling uses of diverging aliases</a></h2> |
| <p>Diverging aliases, like ambiguous aliases, are normalized to inference variables. As normalizing diverging aliases results in trait solver cycles, it always results in an error in the old solver. In the new solver it only results in an error if we wind up requiring all goals to hold in the current context. E.g. normalizing diverging aliases during HIR typeck will result in an error in both solvers.</p> |
| <p>Alias well formedness doesn't require that the alias doesn't diverge<sup class="footnote-reference" id="fr-4-1"><a href="#footnote-4">4</a></sup>, this means that checking an alias is well formed isn't sufficient to cause an error to be emitted for diverging aliases; actually attempting to normalize the alias is required.</p> |
| <p>Erroring on diverging aliases being a side effect of normalization means that it is very <em>arbitrary</em> whether we actually emit an error, it also differs between the old and new solver as we now normalize in less places.</p> |
| <p>An example of the ad-hoc nature of erroring on diverging aliases causing "problems":</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait { |
| type Diverges<D: Trait>; |
| } |
| |
| impl<T> Trait for T { |
| type Diverges<D: Trait> = D::Diverges<D>; |
| } |
| |
| struct Bar<T: ?Sized = <u8 as Trait>::Diverges<u8>>(Box<T>); |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example a diverging alias is used but we happen to not emit an error as we never explicitly normalize the defaults of generic parameters. If the <code>?Sized</code> opt out is removed then an error is emitted because we wind up happening to normalize a <code><u8 as Trait>::Diverges<u8>: Sized</code> goal which as a side effect results in erroring about the diverging alias.</p> |
| <p>Const aliases differ from type aliases a bit here; well formedness of const aliases requires that they can be successfully evaluated (via <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.ClauseKind.html#variant.ConstEvaluatable"><code>ConstEvaluatable</code></a> goals). This means that simply checking well formedness of const arguments is sufficient to error if they would fail to evaluate. It is somewhat unclear whether it would make sense to adopt this for type aliases too or if const aliases should stop requiring this for well formedness<sup class="footnote-reference" id="fr-5-1"><a href="#footnote-5">5</a></sup>.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>In the new solver this is done implicitly <a href="#fr-1-1">↩</a></p> |
| </li> |
| <li id="footnote-2"> |
| <p>There is a subtle difference in how ambiguous aliases in binders are handled between old and new solver. In the old solver we fail to error on some ambiguous aliases inside of higher ranked types whereas the new solver correctly errors. <a href="#fr-2-1">↩</a></p> |
| </li> |
| <li id="footnote-3"> |
| <p>Ambiguous aliases inside of binders cannot be normalized to inference variables, this will be covered more later. <a href="#fr-3-1">↩</a></p> |
| </li> |
| <li id="footnote-4"> |
| <p>As checking aliases are non-diverging cannot be done until they are fully concrete, this would either imply that we cant check aliases are well formed before codegen/const-evaluation or that aliases would go from being well-formed to not well-formed after monomorphization. <a href="#fr-4-1">↩</a></p> |
| </li> |
| <li id="footnote-5"> |
| <p>Const aliases certainly wouldn't be <em>less</em> sound than type aliases if we stopped doing this <a href="#fr-5-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="typingparameter-environments"><a class="header" href="#typingparameter-environments">Typing/Parameter Environments</a></h1> |
| <h2 id="typing-environments"><a class="header" href="#typing-environments">Typing Environments</a></h2> |
| <p>When interacting with the type system there are a few variables to consider that can affect the results of trait solving. |
| The set of in-scope where clauses, and what phase of the compiler type system operations are being performed in (the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html"><code>ParamEnv</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TypingMode.html"><code>TypingMode</code></a> structs respectively).</p> |
| <p>When an environment to perform type system operations in has not yet been created, |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypingEnv.html"><code>TypingEnv</code></a> can be used to bundle all of the external context required into a single type.</p> |
| <p>Once a context to perform type system operations in has been created (e.g. an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/struct.ObligationCtxt.html"><code>ObligationCtxt</code></a> or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html"><code>FnCtxt</code></a>) a <code>TypingEnv</code> is typically not stored anywhere as only the <code>TypingMode</code> is a property of the whole environment, |
| whereas different <code>ParamEnv</code>s can be used on a per-goal basis.</p> |
| <h2 id="parameter-environments"><a class="header" href="#parameter-environments">Parameter Environments</a></h2> |
| <h3 id="what-is-a-paramenv"><a class="header" href="#what-is-a-paramenv">What is a <code>ParamEnv</code></a></h3> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html"><code>ParamEnv</code></a> is a list of in-scope where-clauses, |
| it typically corresponds to a specific item's where clauses. |
| Some clauses are not explicitly written but are instead implicitly added in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/collect/predicates_of/fn.predicates_of.html"><code>predicates_of</code></a> query, |
| such as <code>ConstArgHasType</code> or (some) implied bounds.</p> |
| <p>In most cases <code>ParamEnv</code>s are initially created via the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.param_env"><code>param_env</code> query</a> which returns a <code>ParamEnv</code> derived from the provided item's where clauses. |
| A <code>ParamEnv</code> can also be created with arbitrary sets of clauses that are not derived from a specific item, |
| such as in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html"><code>compare_method_predicate_entailment</code></a> where we create a hybrid <code>ParamEnv</code> consisting of the impl's where clauses and the trait definition's function's where clauses.</p> |
| <hr /> |
| <p>If we have a function such as:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// `foo` would have a `ParamEnv` of: |
| // `[T: Sized, T: Trait, <T as Trait>::Assoc: Clone]` |
| fn foo<T: Trait>() |
| where |
| <T as Trait>::Assoc: Clone, |
| {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>If we were conceptually inside of <code>foo</code> (for example, type-checking or linting it) we would use this <code>ParamEnv</code> everywhere that we interact with the type system. |
| This would allow things such as <a href="normalization.html">normalization</a>, evaluating generic constants, |
| and proving where clauses/goals, to rely on <code>T</code> being sized, implementing <code>Trait</code>, etc.</p> |
| <p>A more concrete example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// `foo` would have a `ParamEnv` of: |
| // `[T: Sized, T: Clone]` |
| fn foo<T: Clone>(a: T) { |
| // when typechecking `foo` we require all the where clauses on `requires_clone` |
| // to hold in order for it to be legal to call. This means we have to |
| // prove `T: Clone`. As we are type checking `foo` we use `foo`'s |
| // environment when trying to check that `T: Clone` holds. |
| // |
| // Trying to prove `T: Clone` with a `ParamEnv` of `[T: Sized, T: Clone]` |
| // will trivially succeed as bound we want to prove is in our environment. |
| requires_clone(a); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Or alternatively an example that would not compile:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// `foo2` would have a `ParamEnv` of: |
| // `[T: Sized]` |
| fn foo2<T>(a: T) { |
| // When typechecking `foo2` we attempt to prove `T: Clone`. |
| // As we are type checking `foo2` we use `foo2`'s environment |
| // when trying to prove `T: Clone`. |
| // |
| // Trying to prove `T: Clone` with a `ParamEnv` of `[T: Sized]` will |
| // fail as there is nothing in the environment telling the trait solver |
| // that `T` implements `Clone` and there exists no user written impl |
| // that could apply. |
| requires_clone(a); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="acquiring-a-paramenv"><a class="header" href="#acquiring-a-paramenv">Acquiring a <code>ParamEnv</code></a></h3> |
| <p>Using the wrong <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html"><code>ParamEnv</code></a> when interacting with the type system can lead to ICEs, |
| illformed programs compiling, or erroring when we shouldn't. |
| See <a href="https://github.com/rust-lang/rust/pull/82159">#82159</a> and <a href="https://github.com/rust-lang/rust/pull/82067">#82067</a> as examples of PRs that modified the compiler to use the correct param env and in the process fixed ICEs.</p> |
| <p>In the large majority of cases, when a <code>ParamEnv</code> is required it either already exists somewhere in scope, |
| or above in the call stack and should be passed down. |
| A non exhaustive list of places where you might find an existing <code>ParamEnv</code>:</p> |
| <ul> |
| <li>During typeck <code>FnCtxt</code> has a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#structfield.param_env"><code>param_env</code> field</a></li> |
| <li>When writing late lints the <code>LateContext</code> has a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/context/struct.LateContext.html#structfield.param_env"><code>param_env</code> field</a></li> |
| <li>During well formedness checking the <code>WfCheckingCtxt</code> has a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/wfcheck/struct.WfCheckingCtxt.html#structfield.param_env"><code>param_env</code> field</a></li> |
| <li>The <code>TypeChecker</code> used for MIR Typeck has a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/struct.TypeChecker.html#structfield.param_env"><code>param_env</code> field</a></li> |
| <li>In the next-gen trait solver all <code>Goal</code>s have a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/canonical/ir/solve/struct.Goal.html#structfield.param_env"><code>param_env</code> field</a> specifying what environment to prove the goal in</li> |
| <li>When editing an existing <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/relate/trait.TypeRelation.html"><code>TypeRelation</code></a> if it implements <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/relate/combine/trait.PredicateEmittingRelation.html"><code>PredicateEmittingRelation</code></a> then a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/trait.PredicateEmittingRelation.html#tymethod.param_env"><code>param_env</code> method</a> will be available.</li> |
| </ul> |
| <p>If you aren't sure if there's a <code>ParamEnv</code> in scope somewhere that can be used it can be worth opening a thread in the <a href="https://rust-lang.zulipchat.com/#narrow/channel/182449-t-compiler.2Fhelp"><code>#t-compiler/help</code></a> Zulip channel where someone may be able to point out where a <code>ParamEnv</code> can be acquired from.</p> |
| <p>Manually constructing a <code>ParamEnv</code> is typically only needed at the start of some kind of top level analysis (e.g. hir typeck or borrow checking). |
| In such cases there are three ways it can be done:</p> |
| <ul> |
| <li>Calling the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#structfield.param_env"><code>tcx.param_env(def_id)</code> query</a> which returns the environment associated with a given definition.</li> |
| <li>Creating an empty environment with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.empty"><code>ParamEnv::empty</code></a>.</li> |
| <li>Using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.new"><code>ParamEnv::new</code></a> to construct an env with an arbitrary set of where clauses. |
| Then calling <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/fn.normalize_param_env_or_error.html"><code>traits::normalize_param_env_or_error</code></a> to handle normalizing and elaborating all the where clauses in the env.</li> |
| </ul> |
| <p>Using the <code>param_env</code> query is by far the most common way to construct a <code>ParamEnv</code> as most of the time the compiler is performing an analysis as part of some specific definition.</p> |
| <p>Creating an empty environment with <code>ParamEnv::empty</code> is typically only done either in codegen (indirectly via <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypingEnv.html#method.fully_monomorphized"><code>TypingEnv::fully_monomorphized</code></a>), |
| or as part of some analysis that do not expect to ever encounter generic parameters |
| (e.g. various parts of coherence/orphan check).</p> |
| <p>Creating an env from an arbitrary set of where clauses is usually unnecessary and should only be done if the environment you need does not correspond to an actual item in the source code (e.g. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html"><code>compare_method_predicate_entailment</code></a>).</p> |
| <h3 id="how-are-paramenvs-constructed"><a class="header" href="#how-are-paramenvs-constructed">How are <code>ParamEnv</code>s constructed</a></h3> |
| <p>Creating a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html"><code>ParamEnv</code></a> is more complicated than simply using the list of where clauses defined on an item as written by the user. |
| We need to both elaborate supertraits into the env and fully normalize all aliases. |
| This logic is handled by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/fn.normalize_param_env_or_error.html"><code>traits::normalize_param_env_or_error</code></a> (even though it does not mention anything about elaboration).</p> |
| <h4 id="elaborating-supertraits"><a class="header" href="#elaborating-supertraits">Elaborating supertraits</a></h4> |
| <p>When we have a function such as <code>fn foo<T: Copy>()</code> we would like to be able to prove <code>T: Clone</code> inside of the function as the <code>Copy</code> trait has a <code>Clone</code> supertrait. |
| Constructing a <code>ParamEnv</code> looks at all of the trait bounds in the env and explicitly adds new where clauses to the <code>ParamEnv</code> for any supertraits found on the traits.</p> |
| <p>A concrete example would be the following function:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait: SuperTrait {} |
| trait SuperTrait: SuperSuperTrait {} |
| |
| // `bar`'s unelaborated `ParamEnv` would be: |
| // `[T: Sized, T: Copy, T: Trait]` |
| fn bar<T: Copy + Trait>(a: T) { |
| requires_impl(a); |
| } |
| |
| fn requires_impl<T: Clone + SuperSuperTrait>(a: T) {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>If we did not elaborate the env then the <code>requires_impl</code> call would fail to typecheck as we would not be able to prove <code>T: Clone</code> or <code>T: SuperSuperTrait</code>. |
| In practice we elaborate the env which means that <code>bar</code>'s <code>ParamEnv</code> is actually: |
| <code>[T: Sized, T: Copy, T: Clone, T: Trait, T: SuperTrait, T: SuperSuperTrait]</code> |
| This allows us to prove <code>T: Clone</code> and <code>T: SuperSuperTrait</code> when type checking <code>bar</code>.</p> |
| <p>The <code>Clone</code> trait has a <code>Sized</code> supertrait however we do not end up with two <code>T: Sized</code> bounds in the env (one for the supertrait and one for the implicitly added <code>T: Sized</code> bound) as the elaboration process (implemented via <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/traits/util/fn.elaborate.html"><code>util::elaborate</code></a>) deduplicates where clauses.</p> |
| <p>A side effect of this is that even if no actual elaboration of supertraits takes place, |
| the existing where clauses in the env are <em>also</em> deduplicated. |
| See the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait {} |
| // The unelaborated `ParamEnv` would be: |
| // `[T: Sized, T: Trait, T: Trait]` |
| // but after elaboration it would be: |
| // `[T: Sized, T: Trait]` |
| fn foo<T: Trait + Trait>() {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>The <a href="./solve/trait-solving.html">next-gen trait solver</a> also requires this elaboration to take place.</p> |
| <h4 id="normalizing-all-bounds"><a class="header" href="#normalizing-all-bounds">Normalizing all bounds</a></h4> |
| <p>In the old trait solver the where clauses stored in <code>ParamEnv</code> are required to be fully normalized as otherwise the trait solver will not function correctly. |
| A concrete example of needing to normalize the <code>ParamEnv</code> is the following:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<T> { |
| type Assoc; |
| } |
| |
| trait Other { |
| type Bar; |
| } |
| |
| impl<T> Other for T { |
| type Bar = u32; |
| } |
| |
| // `foo`'s unnormalized `ParamEnv` would be: |
| // `[T: Sized, U: Sized, U: Trait<T::Bar>]` |
| fn foo<T, U>(a: U) |
| where |
| U: Trait<<T as Other>::Bar>, |
| { |
| requires_impl(a); |
| } |
| |
| fn requires_impl<U: Trait<u32>>(_: U) {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>As humans we can tell that <code><T as Other>::Bar</code> is equal to <code>u32</code> so the trait bound on <code>U</code> is equivalent to <code>U: Trait<u32></code>. |
| In practice trying to prove <code>U: Trait<u32></code> in the old solver in this environment would fail as it is unable to determine that <code><T as Other>::Bar</code> is equal to <code>u32</code>.</p> |
| <p>To work around this we normalize <code>ParamEnv</code>'s after constructing them so that <code>foo</code>'s <code>ParamEnv</code> is actually: <code>[T: Sized, U: Sized, U: Trait<u32>]</code> which means the trait solver is now able to use the <code>U: Trait<u32></code> in the <code>ParamEnv</code> to determine that the trait bound <code>U: Trait<u32></code> holds.</p> |
| <p>This workaround does not work in all cases as normalizing associated types requires a <code>ParamEnv</code> which introduces a bootstrapping problem. |
| We need a normalized <code>ParamEnv</code> in order for normalization to give correct results, but we need to normalize to get that <code>ParamEnv</code>. |
| Currently we normalize the <code>ParamEnv</code> once using the unnormalized param env and it tends to give okay results in practice even though there are some examples where this breaks (<a href="https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=e6933265ea3e84eaa47019465739992c">example</a>).</p> |
| <p>In the next-gen trait solver the requirement for all where clauses in the <code>ParamEnv</code> to be fully normalized is not present and so we do not normalize when constructing <code>ParamEnv</code>s.</p> |
| <h2 id="typing-modes"><a class="header" href="#typing-modes">Typing Modes</a></h2> |
| <p>Depending on what context we are performing type system operations in, |
| different behaviour may be required. |
| For example during coherence there are stronger requirements about when we can consider goals to not hold or when we can consider types to be unequal.</p> |
| <p>Tracking which "phase" of the compiler type system operations are being performed in is done by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TypingMode.html"><code>TypingMode</code></a> enum. |
| The documentation on the <code>TypingMode</code> enum is quite good so instead of repeating it here verbatim we would recommend reading the API documentation directly.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="type-inference"><a class="header" href="#type-inference">Type inference</a></h1> |
| <p>Type inference is the process of automatic detection of the type of an |
| expression.</p> |
| <p>It is what allows Rust to work with fewer or no type annotations, |
| making things easier for users:</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let mut things = vec![]; |
| things.push("thing"); |
| }</code></pre></pre> |
| <p>Here, the type of <code>things</code> is <em>inferred</em> to be <code>Vec<&str></code> because of the value |
| we push into <code>things</code>.</p> |
| <p>The type inference is based on the standard Hindley-Milner (HM) type inference |
| algorithm, but extended in various ways to accommodate subtyping, region |
| inference, and higher-ranked types.</p> |
| <h2 id="a-note-on-terminology"><a class="header" href="#a-note-on-terminology">A note on terminology</a></h2> |
| <p>We use the notation <code>?T</code> to refer to inference variables, also called |
| existential variables.</p> |
| <p>We use the terms "region" and "lifetime" interchangeably. Both refer to |
| the <code>'a</code> in <code>&'a T</code>.</p> |
| <p>The term "bound region" refers to a region that is bound in a function |
| signature, such as the <code>'a</code> in <code>for<'a> fn(&'a u32)</code>. A region is |
| "free" if it is not bound.</p> |
| <h2 id="creating-an-inference-context"><a class="header" href="#creating-an-inference-context">Creating an inference context</a></h2> |
| <p>You create an inference context by doing something like |
| the following:</p> |
| <pre><code class="language-rust ignore">let infcx = tcx.infer_ctxt().build(); |
| // Use the inference context `infcx` here.</code></pre> |
| <p><code>infcx</code> has the type <code>InferCtxt<'tcx></code>, the same <code>'tcx</code> lifetime as on |
| the <code>tcx</code> it was built from.</p> |
| <p>The <code>tcx.infer_ctxt</code> method actually returns a builder, which means |
| there are some kinds of configuration you can do before the <code>infcx</code> is |
| created. See <code>InferCtxtBuilder</code> for more information.</p> |
| <p><a id="vars"></a></p> |
| <h2 id="inference-variables"><a class="header" href="#inference-variables">Inference variables</a></h2> |
| <p>The main purpose of the inference context is to house a bunch of |
| <strong>inference variables</strong> – these represent types or regions whose precise |
| value is not yet known, but will be uncovered as we perform type-checking.</p> |
| <p>If you're familiar with the basic ideas of unification from H-M type |
| systems, or logic languages like Prolog, this is the same concept. If |
| you're not, you might want to read a tutorial on how H-M type |
| inference works, or perhaps this blog post on |
| <a href="http://smallcultfollowing.com/babysteps/blog/2017/03/25/unification-in-chalk-part-1/">unification in the Chalk project</a>.</p> |
| <p>All told, the inference context stores five kinds of inference variables |
| (as of <!-- date-check --> March 2023):</p> |
| <ul> |
| <li>Type variables, which come in three varieties: |
| <ul> |
| <li>General type variables (the most common). These can be unified with any |
| type.</li> |
| <li>Integral type variables, which can only be unified with an integral type, |
| and arise from an integer literal expression like <code>22</code>.</li> |
| <li>Float type variables, which can only be unified with a float type, and |
| arise from a float literal expression like <code>22.0</code>.</li> |
| </ul> |
| </li> |
| <li>Region variables, which represent lifetimes, and arise all over the place.</li> |
| <li>Const variables, which represent constants.</li> |
| </ul> |
| <p>All the type variables work in much the same way: you can create a new |
| type variable, and what you get is <code>Ty<'tcx></code> representing an |
| unresolved type <code>?T</code>. Then later you can apply the various operations |
| that the inferencer supports, such as equality or subtyping, and it |
| will possibly <strong>instantiate</strong> (or <strong>bind</strong>) that <code>?T</code> to a specific |
| value as a result.</p> |
| <p>The region variables work somewhat differently, and are described |
| below in a separate section.</p> |
| <h2 id="enforcing-equality--subtyping"><a class="header" href="#enforcing-equality--subtyping">Enforcing equality / subtyping</a></h2> |
| <p>The most basic operations you can perform in the type inferencer is |
| <strong>equality</strong>, which forces two types <code>T</code> and <code>U</code> to be the same. The |
| recommended way to add an equality constraint is to use the <code>at</code> |
| method, roughly like so:</p> |
| <pre><code class="language-rust ignore">infcx.at(...).eq(t, u);</code></pre> |
| <p>The first <code>at()</code> call provides a bit of context, i.e. why you are |
| doing this unification, and in what environment, and the <code>eq</code> method |
| performs the actual equality constraint.</p> |
| <p>When you equate things, you force them to be precisely equal. Equating |
| returns an <code>InferResult</code> – if it returns <code>Err(err)</code>, then equating |
| failed, and the enclosing <code>TypeError</code> will tell you what went wrong.</p> |
| <p>The success case is perhaps more interesting. The "primary" return |
| type of <code>eq</code> is <code>()</code> – that is, when it succeeds, it doesn't return a |
| value of any particular interest. Rather, it is executed for its |
| side-effects of constraining type variables and so forth. However, the |
| actual return type is not <code>()</code>, but rather <code>InferOk<()></code>. The |
| <code>InferOk</code> type is used to carry extra trait obligations – your job is |
| to ensure that these are fulfilled (typically by enrolling them in a |
| fulfillment context). See the <a href="traits/resolution.html">trait chapter</a> for more background on that.</p> |
| <p>You can similarly enforce subtyping through <code>infcx.at(..).sub(..)</code>. The same |
| basic concepts as above apply.</p> |
| <h2 id="trying-equality"><a class="header" href="#trying-equality">"Trying" equality</a></h2> |
| <p>Sometimes you would like to know if it is <em>possible</em> to equate two |
| types without error. You can test that with <code>infcx.can_eq</code> (or |
| <code>infcx.can_sub</code> for subtyping). If this returns <code>Ok</code>, then equality |
| is possible – but in all cases, any side-effects are reversed.</p> |
| <p>Be aware, though, that the success or failure of these methods is always |
| <strong>modulo regions</strong>. That is, two types <code>&'a u32</code> and <code>&'b u32</code> will |
| return <code>Ok</code> for <code>can_eq</code>, even if <code>'a != 'b</code>. This falls out from the |
| "two-phase" nature of how we solve region constraints.</p> |
| <h2 id="snapshots"><a class="header" href="#snapshots">Snapshots</a></h2> |
| <p>As described in the previous section on <code>can_eq</code>, often it is useful |
| to be able to do a series of operations and then roll back their |
| side-effects. This is done for various reasons: one of them is to be |
| able to backtrack, trying out multiple possibilities before settling |
| on which path to take. Another is in order to ensure that a series of |
| smaller changes take place atomically or not at all.</p> |
| <p>To allow for this, the inference context supports a <code>snapshot</code> method. |
| When you call it, it will start recording changes that occur from the |
| operations you perform. When you are done, you can either invoke |
| <code>rollback_to</code>, which will undo those changes, or else <code>confirm</code>, which |
| will make them permanent. Snapshots can be nested as long as you follow |
| a stack-like discipline.</p> |
| <p>Rather than use snapshots directly, it is often helpful to use the |
| methods like <code>commit_if_ok</code> or <code>probe</code> that encapsulate higher-level |
| patterns.</p> |
| <h2 id="subtyping-obligations"><a class="header" href="#subtyping-obligations">Subtyping obligations</a></h2> |
| <p>One thing worth discussing is subtyping obligations. When you force |
| two types to be a subtype, like <code>?T <: i32</code>, we can often convert those |
| into equality constraints. This follows from Rust's rather limited notion |
| of subtyping: so, in the above case, <code>?T <: i32</code> is equivalent to <code>?T = i32</code>.</p> |
| <p>However, in some cases we have to be more careful. For example, when |
| regions are involved. So if you have <code>?T <: &'a i32</code>, what we would do |
| is to first "generalize" <code>&'a i32</code> into a type with a region variable: |
| <code>&'?b i32</code>, and then unify <code>?T</code> with that (<code>?T = &'?b i32</code>). We then |
| relate this new variable with the original bound:</p> |
| <pre><code class="language-text">&'?b i32 <: &'a i32 |
| </code></pre> |
| <p>This will result in a region constraint (see below) of <code>'?b: 'a</code>.</p> |
| <p>One final interesting case is relating two unbound type variables, |
| like <code>?T <: ?U</code>. In that case, we can't make progress, so we enqueue |
| an obligation <code>Subtype(?T, ?U)</code> and return it via the <code>InferOk</code> |
| mechanism. You'll have to try again when more details about <code>?T</code> or |
| <code>?U</code> are known.</p> |
| <h2 id="region-constraints"><a class="header" href="#region-constraints">Region constraints</a></h2> |
| <p>Regions are inferenced somewhat differently from types. Rather than |
| eagerly unifying things, we simply collect constraints as we go, but |
| make (almost) no attempt to solve regions. These constraints have the |
| form of an "outlives" constraint:</p> |
| <pre><code class="language-text">'a: 'b |
| </code></pre> |
| <p>Actually the code tends to view them as a subregion relation, but it's the same |
| idea:</p> |
| <pre><code class="language-text">'b <= 'a |
| </code></pre> |
| <p>(There are various other kinds of constraints, such as "verifys"; see |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/region_constraints/index.html"><code>region_constraints</code></a> module for details.)</p> |
| <p>There is one case where we do some amount of eager unification. If you have an |
| equality constraint between two regions</p> |
| <pre><code class="language-text">'a = 'b |
| </code></pre> |
| <p>we will record that fact in a unification table. You can then use |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/region_constraints/struct.RegionConstraintCollector.html#method.opportunistic_resolve_var"><code>opportunistic_resolve_var</code></a> to convert <code>'b</code> to <code>'a</code> (or vice |
| versa). This is sometimes needed to ensure termination of fixed-point |
| algorithms.</p> |
| <h2 id="solving-region-constraints"><a class="header" href="#solving-region-constraints">Solving region constraints</a></h2> |
| <p>Region constraints are only solved at the very end of |
| typechecking, once all other constraints are known and |
| all other obligations have been proven. There are two |
| ways to solve region constraints right now: lexical and |
| non-lexical. Eventually there will only be one.</p> |
| <p>An exception here is the leak-check which is used during trait solving |
| and relies on region constraints containing higher-ranked regions. Region |
| constraints in the root universe (i.e. not arising from a <code>for<'a></code>) must |
| not influence the trait system, as these regions are all erased during |
| codegen.</p> |
| <p>To solve <strong>lexical</strong> region constraints, you invoke |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/struct.ObligationCtxt.html#method.resolve_regions_and_report_errors"><code>resolve_regions_and_report_errors</code></a>. This "closes" the region |
| constraint process and invokes the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/lexical_region_resolve/index.html"><code>lexical_region_resolve</code></a> code. Once |
| this is done, any further attempt to equate or create a subtyping |
| relationship will yield an ICE.</p> |
| <p>The NLL solver (actually, the MIR type-checker) does things slightly |
| differently. It uses canonical queries for trait solving which use |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/struct.InferCtxt.html#method.take_and_reset_region_constraints"><code>take_and_reset_region_constraints</code></a> at the end. This extracts all of the |
| outlives constraints added during the canonical query. This is required |
| as the NLL solver must not only know <em>what</em> regions outlive each other, |
| but also <em>where</em>. Finally, the NLL solver invokes <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/struct.InferCtxt.html#method.get_region_var_infos"><code>get_region_var_infos</code></a>, |
| providing all region variables to the solver.</p> |
| <h2 id="lexical-region-resolution"><a class="header" href="#lexical-region-resolution">Lexical region resolution</a></h2> |
| <p>Lexical region resolution is done by initially assigning each region |
| variable to an empty value. We then process each outlives constraint |
| repeatedly, growing region variables until a fixed-point is reached. |
| Region variables can be grown using a least-upper-bound relation on |
| the region lattice in a fairly straightforward fashion.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="trait-resolution-old-style"><a class="header" href="#trait-resolution-old-style">Trait resolution (old-style)</a></h1> |
| <p>This chapter describes the general process of <em>trait resolution</em> and points out |
| some non-obvious things.</p> |
| <p><strong>Note:</strong> This chapter (and its subchapters) describe how the trait |
| solver <strong>currently</strong> works. However, we are in the process of |
| designing a new trait solver. If you'd prefer to read about <em>that</em>, |
| see <a href="traits/./chalk.html"><em>this</em> subchapter</a>.</p> |
| <h2 id="major-concepts"><a class="header" href="#major-concepts">Major concepts</a></h2> |
| <p>Trait resolution is the process of pairing up an impl with each |
| reference to a trait. So, for example, if there is a generic function like:</p> |
| <pre><code class="language-rust ignore">fn clone_slice<T:Clone>(x: &[T]) -> Vec<T> { ... }</code></pre> |
| <p>and then a call to that function:</p> |
| <pre><code class="language-rust ignore">let v: Vec<isize> = clone_slice(&[1, 2, 3])</code></pre> |
| <p>it is the job of trait resolution to figure out whether there exists an impl of |
| (in this case) <code>isize : Clone</code>.</p> |
| <p>Note that in some cases, like generic functions, we may not be able to |
| find a specific impl, but we can figure out that the caller must |
| provide an impl. For example, consider the body of <code>clone_slice</code>:</p> |
| <pre><code class="language-rust ignore">fn clone_slice<T:Clone>(x: &[T]) -> Vec<T> { |
| let mut v = Vec::new(); |
| for e in &x { |
| v.push((*e).clone()); // (*) |
| } |
| }</code></pre> |
| <p>The line marked <code>(*)</code> is only legal if <code>T</code> (the type of <code>*e</code>) |
| implements the <code>Clone</code> trait. Naturally, since we don't know what <code>T</code> |
| is, we can't find the specific impl; but based on the bound <code>T:Clone</code>, |
| we can say that there exists an impl which the caller must provide.</p> |
| <p>We use the term <em>obligation</em> to refer to a trait reference in need of |
| an impl. Basically, the trait resolution system resolves an obligation |
| by proving that an appropriate impl does exist.</p> |
| <p>During type checking, we do not store the results of trait selection. |
| We simply wish to verify that trait selection will succeed. Then |
| later, at codegen time, when we have all concrete types available, we |
| can repeat the trait selection to choose an actual implementation, which |
| will then be generated in the output binary.</p> |
| <h2 id="overview-6"><a class="header" href="#overview-6">Overview</a></h2> |
| <p>Trait resolution consists of three major parts:</p> |
| <ul> |
| <li> |
| <p><strong>Selection</strong>: Deciding how to resolve a specific obligation. For |
| example, selection might decide that a specific obligation can be |
| resolved by employing an impl which matches the <code>Self</code> type, or by using a |
| parameter bound (e.g. <code>T: Trait</code>). In the case of an impl, selecting one |
| obligation can create <em>nested obligations</em> because of where clauses |
| on the impl itself. It may also require evaluating those nested |
| obligations to resolve ambiguities.</p> |
| </li> |
| <li> |
| <p><strong>Fulfillment</strong>: The fulfillment code is what tracks that obligations |
| are completely fulfilled. Basically it is a worklist of obligations |
| to be selected: once selection is successful, the obligation is |
| removed from the worklist and any nested obligations are enqueued. |
| Fulfillment constrains inference variables.</p> |
| </li> |
| <li> |
| <p><strong>Evaluation</strong>: Checks whether obligations holds without constraining |
| any inference variables. Used by selection.</p> |
| </li> |
| </ul> |
| <h2 id="selection"><a class="header" href="#selection">Selection</a></h2> |
| <p>Selection is the process of deciding whether an obligation can be |
| resolved and, if so, how it is to be resolved (via impl, where clause, etc). |
| The main interface is the <code>select()</code> function, which takes an obligation |
| and returns a <code>SelectionResult</code>. There are three possible outcomes:</p> |
| <ul> |
| <li> |
| <p><code>Ok(Some(selection))</code> – yes, the obligation can be resolved, and |
| <code>selection</code> indicates how. If the impl was resolved via an impl, |
| then <code>selection</code> may also indicate nested obligations that are required |
| by the impl.</p> |
| </li> |
| <li> |
| <p><code>Ok(None)</code> – we are not yet sure whether the obligation can be |
| resolved or not. This happens most commonly when the obligation |
| contains unbound type variables.</p> |
| </li> |
| <li> |
| <p><code>Err(err)</code> – the obligation definitely cannot be resolved due to a |
| type error or because there are no impls that could possibly apply.</p> |
| </li> |
| </ul> |
| <p>The basic algorithm for selection is broken into two big phases: |
| candidate assembly and confirmation.</p> |
| <p>Note that because of how lifetime inference works, it is not possible to |
| give back immediate feedback as to whether a unification or subtype |
| relationship between lifetimes holds or not. Therefore, lifetime |
| matching is <em>not</em> considered during selection. This is reflected in |
| the fact that subregion assignment is infallible. This may yield |
| lifetime constraints that will later be found to be in error (in |
| contrast, the non-lifetime-constraints have already been checked |
| during selection and can never cause an error, though naturally they |
| may lead to other errors downstream).</p> |
| <h3 id="candidate-assembly"><a class="header" href="#candidate-assembly">Candidate assembly</a></h3> |
| <p><strong>TODO</strong>: Talk about <em>why</em> we have different candidates, and why it needs to happen in a probe.</p> |
| <p>Searches for impls/where-clauses/etc that might |
| possibly be used to satisfy the obligation. Each of those is called |
| a candidate. To avoid ambiguity, we want to find exactly one |
| candidate that is definitively applicable. In some cases, we may not |
| know whether an impl/where-clause applies or not – this occurs when |
| the obligation contains unbound inference variables.</p> |
| <p>The subroutines that decide whether a particular impl/where-clause/etc applies |
| to a particular obligation are collectively referred to as the process of |
| <em>matching</em>. For <code>impl</code> candidates <!-- date-check: Oct 2022 -->, |
| this amounts to unifying the impl header (the <code>Self</code> type and the trait arguments) |
| while ignoring nested obligations. If matching succeeds then we add it |
| to a set of candidates. There are other rules when assembling candidates for |
| built-in traits such as <code>Copy</code>, <code>Sized</code>, and <code>CoerceUnsized</code>.</p> |
| <p>Once this first pass is done, we can examine the set of candidates. If |
| it is a singleton set, then we are done: this is the only impl in |
| scope that could possibly apply. Otherwise, we can <strong>winnow</strong> down the set |
| of candidates by using where clauses and other conditions. Winnowing uses |
| <code>evaluate_candidate</code> to check whether the nested obligations may apply. |
| If this still leaves more than 1 candidate, we use <code> fn candidate_should_be_dropped_in_favor_of</code> |
| to prefer some candidates over others.</p> |
| <p>If this reduced set yields a single, unambiguous entry, we're good to go, |
| otherwise the result is considered ambiguous.</p> |
| <h4 id="winnowing-resolving-ambiguities"><a class="header" href="#winnowing-resolving-ambiguities">Winnowing: Resolving ambiguities</a></h4> |
| <p>But what happens if there are multiple impls where all the types |
| unify? Consider this example:</p> |
| <pre><code class="language-rust ignore">trait Get { |
| fn get(&self) -> Self; |
| } |
| |
| impl<T: Copy> Get for T { |
| fn get(&self) -> T { |
| *self |
| } |
| } |
| |
| impl<T: Get> Get for Box<T> { |
| fn get(&self) -> Box<T> { |
| Box::new(<T>::get(self)) |
| } |
| }</code></pre> |
| <p>What happens when we invoke <code>get(&Box::new(1_u16))</code>, for example? In this |
| case, the <code>Self</code> type is <code>Box<u16></code> – that unifies with both impls, |
| because the first applies to all types <code>T</code>, and the second to all |
| <code>Box<T></code>. In order for this to be unambiguous, the compiler does a <em>winnowing</em> |
| pass that considers <code>where</code> clauses |
| and attempts to remove candidates. In this case, the first impl only |
| applies if <code>Box<u16> : Copy</code>, which doesn't hold. After winnowing, |
| then, we are left with just one candidate, so we can proceed.</p> |
| <h4 id="where-clauses"><a class="header" href="#where-clauses"><code>where</code> clauses</a></h4> |
| <p>Besides an impl, the other major way to resolve an obligation is via a |
| where clause. The selection process is always given a <a href="traits/../typing_parameter_envs.html">parameter |
| environment</a> which contains a list of where clauses, which are |
| basically obligations that we can assume are satisfiable. We will iterate |
| over that list and check whether our current obligation can be found |
| in that list. If so, it is considered satisfied. More precisely, we |
| want to check whether there is a where-clause obligation that is for |
| the same trait (or some subtrait) and which can match against the obligation.</p> |
| <p>Consider this simple example:</p> |
| <pre><code class="language-rust ignore">trait A1 { |
| fn do_a1(&self); |
| } |
| trait A2 : A1 { ... } |
| |
| trait B { |
| fn do_b(&self); |
| } |
| |
| fn foo<X:A2+B>(x: X) { |
| x.do_a1(); // (*) |
| x.do_b(); // (#) |
| }</code></pre> |
| <p>In the body of <code>foo</code>, clearly we can use methods of <code>A1</code>, <code>A2</code>, or <code>B</code> |
| on variable <code>x</code>. The line marked <code>(*)</code> will incur an obligation <code>X: A1</code>, |
| while the line marked <code>(#)</code> will incur an obligation <code>X: B</code>. Meanwhile, |
| the parameter environment will contain two where-clauses: <code>X : A2</code> and <code>X : B</code>. |
| For each obligation, then, we search this list of where-clauses. The |
| obligation <code>X: B</code> trivially matches against the where-clause <code>X: B</code>. |
| To resolve an obligation <code>X:A1</code>, we would note that <code>X:A2</code> implies that <code>X:A1</code>.</p> |
| <h3 id="confirmation"><a class="header" href="#confirmation">Confirmation</a></h3> |
| <p><em>Confirmation</em> unifies the output type parameters of the trait with the |
| values found in the obligation, possibly yielding a type error.</p> |
| <p>Suppose we have the following variation of the <code>Convert</code> example in the |
| previous section:</p> |
| <pre><code class="language-rust ignore">trait Convert<Target> { |
| fn convert(&self) -> Target; |
| } |
| |
| impl Convert<usize> for isize { ... } // isize -> usize |
| impl Convert<isize> for usize { ... } // usize -> isize |
| |
| let x: isize = ...; |
| let y: char = x.convert(); // NOTE: `y: char` now!</code></pre> |
| <p>Confirmation is where an error would be reported because the impl specified |
| that <code>Target</code> would be <code>usize</code>, but the obligation reported <code>char</code>. Hence the |
| result of selection would be an error.</p> |
| <p>Note that the candidate impl is chosen based on the <code>Self</code> type, but |
| confirmation is done based on (in this case) the <code>Target</code> type parameter.</p> |
| <h3 id="selection-during-codegen"><a class="header" href="#selection-during-codegen">Selection during codegen</a></h3> |
| <p>As mentioned above, during type checking, we do not store the results of trait |
| selection. At codegen time, we repeat the trait selection to choose a particular |
| impl for each method call. This is done using <code>fn codegen_select_candidate</code>. |
| In this second selection, we do not consider any where-clauses to be in scope |
| because we know that each resolution will resolve to a particular impl.</p> |
| <p>One interesting twist has to do with nested obligations. In general, in codegen, |
| we only need to figure out which candidate applies, and we do not care about nested obligations, |
| as these are already assumed to be true. Nonetheless, we <em>do</em> currently fulfill all of them. |
| That is because it can sometimes inform the results of type inference. |
| That is, we do not have the full substitutions in terms of the type variables |
| of the impl available to us, so we must run trait selection to figure |
| everything out.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="higher-ranked-trait-bounds"><a class="header" href="#higher-ranked-trait-bounds">Higher-ranked trait bounds</a></h1> |
| <p>One of the more subtle concepts in trait resolution is <em>higher-ranked trait |
| bounds</em>. An example of such a bound is <code>for<'a> MyTrait<&'a isize></code>. |
| Let's walk through how selection on higher-ranked trait references |
| works.</p> |
| <h2 id="basic-matching-and-placeholder-leaks"><a class="header" href="#basic-matching-and-placeholder-leaks">Basic matching and placeholder leaks</a></h2> |
| <p>Suppose we have a trait <code>Foo</code>:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo<X> { |
| fn foo(&self, x: X) { } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Let's say we have a function <code>want_hrtb</code> that wants a type which |
| implements <code>Foo<&'a isize></code> for any <code>'a</code>:</p> |
| <pre><code class="language-rust ignore">fn want_hrtb<T>() where T : for<'a> Foo<&'a isize> { ... }</code></pre> |
| <p>Now we have a struct <code>AnyInt</code> that implements <code>Foo<&'a isize></code> for any |
| <code>'a</code>:</p> |
| <pre><code class="language-rust ignore">struct AnyInt; |
| impl<'a> Foo<&'a isize> for AnyInt { }</code></pre> |
| <p>And the question is, does <code>AnyInt : for<'a> Foo<&'a isize></code>? We want the |
| answer to be yes. The algorithm for figuring it out is closely related |
| to the subtyping for higher-ranked types (which is described <a href="traits/./hrtb.html">here</a> |
| and also in a <a href="https://www.microsoft.com/en-us/research/publication/practical-type-inference-for-arbitrary-rank-types">paper by SPJ</a>. If you wish to understand higher-ranked |
| subtyping, we recommend you read the paper). There are a few parts:</p> |
| <ol> |
| <li>Replace bound regions in the obligation with placeholders.</li> |
| <li>Match the impl against the <a href="traits/../appendix/glossary.html#placeholder">placeholder</a> obligation.</li> |
| <li>Check for <em>placeholder leaks</em>.</li> |
| </ol> |
| <p>So let's work through our example.</p> |
| <ol> |
| <li> |
| <p>The first thing we would do is to |
| replace the bound region in the obligation with a placeholder, yielding |
| <code>AnyInt : Foo<&'0 isize></code> (here <code>'0</code> represents placeholder region #0). |
| Note that we now have no quantifiers; |
| in terms of the compiler type, this changes from a <code>ty::PolyTraitRef</code> |
| to a <code>TraitRef</code>. We would then create the <code>TraitRef</code> from the impl, |
| using fresh variables for it's bound regions (and thus getting |
| <code>Foo<&'$a isize></code>, where <code>'$a</code> is the inference variable for <code>'a</code>).</p> |
| </li> |
| <li> |
| <p>Next |
| we relate the two trait refs, yielding a graph with the constraint |
| that <code>'0 == '$a</code>.</p> |
| </li> |
| <li> |
| <p>Finally, we check for placeholder "leaks" – a |
| leak is basically any attempt to relate a placeholder region to another |
| placeholder region, or to any region that pre-existed the impl match. |
| The leak check is done by searching from the placeholder region to find |
| the set of regions that it is related to in any way. This is called |
| the "taint" set. To pass the check, that set must consist <em>solely</em> of |
| itself and region variables from the impl. If the taint set includes |
| any other region, then the match is a failure. In this case, the taint |
| set for <code>'0</code> is <code>{'0, '$a}</code>, and hence the check will succeed.</p> |
| </li> |
| </ol> |
| <p>Let's consider a failure case. Imagine we also have a struct</p> |
| <pre><code class="language-rust ignore">struct StaticInt; |
| impl Foo<&'static isize> for StaticInt;</code></pre> |
| <p>We want the obligation <code>StaticInt : for<'a> Foo<&'a isize></code> to be |
| considered unsatisfied. The check begins just as before. <code>'a</code> is |
| replaced with a placeholder <code>'0</code> and the impl trait reference is instantiated to |
| <code>Foo<&'static isize></code>. When we relate those two, we get a constraint |
| like <code>'static == '0</code>. This means that the taint set for <code>'0</code> is <code>{'0, 'static}</code>, which fails the leak check.</p> |
| <p><strong>TODO</strong>: This is because <code>'static</code> is not a region variable but is in the |
| taint set, right?</p> |
| <h2 id="higher-ranked-trait-obligations"><a class="header" href="#higher-ranked-trait-obligations">Higher-ranked trait obligations</a></h2> |
| <p>Once the basic matching is done, we get to another interesting topic: |
| how to deal with impl obligations. I'll work through a simple example |
| here. Imagine we have the traits <code>Foo</code> and <code>Bar</code> and an associated impl:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo<X> { |
| fn foo(&self, x: X) { } |
| } |
| |
| trait Bar<X> { |
| fn bar(&self, x: X) { } |
| } |
| |
| impl<X,F> Foo<X> for F |
| where F : Bar<X> |
| { |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Now let's say we have an obligation <code>Baz: for<'a> Foo<&'a isize></code> and we match |
| this impl. What obligation is generated as a result? We want to get |
| <code>Baz: for<'a> Bar<&'a isize></code>, but how does that happen?</p> |
| <p>After the matching, we are in a position where we have a placeholder |
| substitution like <code>X => &'0 isize</code>. If we apply this substitution to the |
| impl obligations, we get <code>F : Bar<&'0 isize></code>. Obviously this is not |
| directly usable because the placeholder region <code>'0</code> cannot leak out of |
| our computation.</p> |
| <p>What we do is to create an inverse mapping from the taint set of <code>'0</code> |
| back to the original bound region (<code>'a</code>, here) that <code>'0</code> resulted |
| from. (This is done in <code>higher_ranked::plug_leaks</code>). We know that the |
| leak check passed, so this taint set consists solely of the placeholder |
| region itself plus various intermediate region variables. We then walk |
| the trait-reference and convert every region in that taint set back to |
| a late-bound region, so in this case we'd wind up with |
| <code>Baz: for<'a> Bar<&'a isize></code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="caching-and-subtle-considerations-therewith"><a class="header" href="#caching-and-subtle-considerations-therewith">Caching and subtle considerations therewith</a></h1> |
| <p>In general, we attempt to cache the results of trait selection. This |
| is a somewhat complex process. Part of the reason for this is that we |
| want to be able to cache results even when all the types in the trait |
| reference are not fully known. In that case, it may happen that the |
| trait selection process is also influencing type variables, so we have |
| to be able to not only cache the <em>result</em> of the selection process, |
| but <em>replay</em> its effects on the type variables.</p> |
| <h2 id="an-example"><a class="header" href="#an-example">An example</a></h2> |
| <p>The high-level idea of how the cache works is that we first replace |
| all unbound inference variables with placeholder versions. Therefore, |
| if we had a trait reference <code>usize : Foo<$t></code>, where <code>$t</code> is an unbound |
| inference variable, we might replace it with <code>usize : Foo<$0></code>, where |
| <code>$0</code> is a placeholder type. We would then look this up in the cache.</p> |
| <p>If we found a hit, the hit would tell us the immediate next step to |
| take in the selection process (e.g. apply impl #22, or apply where |
| clause <code>X : Foo<Y></code>).</p> |
| <p>On the other hand, if there is no hit, we need to go through the <a href="traits/./resolution.html#selection">selection |
| process</a> from scratch. Suppose, we come to the conclusion that the only |
| possible impl is this one, with def-id 22:</p> |
| <pre><code class="language-rust ignore">impl Foo<isize> for usize { ... } // Impl #22</code></pre> |
| <p>We would then record in the cache <code>usize : Foo<$0> => ImplCandidate(22)</code>. Next |
| we would <a href="traits/./resolution.html#confirmation">confirm</a> <code>ImplCandidate(22)</code>, which would (as a side-effect) unify |
| <code>$t</code> with <code>isize</code>.</p> |
| <p>Now, at some later time, we might come along and see a <code>usize : Foo<$u></code>. When replaced with a placeholder, this would yield <code>usize : Foo<$0></code>, just as |
| before, and hence the cache lookup would succeed, yielding |
| <code>ImplCandidate(22)</code>. We would confirm <code>ImplCandidate(22)</code> which would |
| (as a side-effect) unify <code>$u</code> with <code>isize</code>.</p> |
| <h2 id="where-clauses-and-the-local-vs-global-cache"><a class="header" href="#where-clauses-and-the-local-vs-global-cache">Where clauses and the local vs global cache</a></h2> |
| <p>One subtle interaction is that the results of trait lookup will vary |
| depending on what where clauses are in scope. Therefore, we actually |
| have <em>two</em> caches, a local and a global cache. The local cache is |
| attached to the <a href="traits/../typing_parameter_envs.html"><code>ParamEnv</code></a>, and the global cache attached to the |
| <a href="traits/../ty.html"><code>tcx</code></a>. We use the local cache whenever the result might depend on the |
| where clauses that are in scope. The determination of which cache to |
| use is done by the method <code>pick_candidate_cache</code> in <code>select.rs</code>. At |
| the moment, we use a very simple, conservative rule: if there are any |
| where-clauses in scope, then we use the local cache. We used to try |
| and draw finer-grained distinctions, but that led to a series of |
| annoying and weird bugs like <a href="https://github.com/rust-lang/rust/issues/22019">#22019</a> and <a href="https://github.com/rust-lang/rust/issues/18290">#18290</a>. This simple rule seems |
| to be pretty clearly safe and also still retains a very high hit rate |
| (~95% when compiling rustc).</p> |
| <p><strong>TODO</strong>: it looks like <code>pick_candidate_cache</code> no longer exists. In |
| general, is this section still accurate at all?</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="implied-bounds"><a class="header" href="#implied-bounds">Implied bounds</a></h1> |
| <p>We currently add implied region bounds to avoid explicit annotations. e.g. |
| <code>fn foo<'a, T>(x: &'a T)</code> can freely assume that <code>T: 'a</code> holds without specifying it.</p> |
| <p>There are two kinds of implied bounds: explicit and implicit. Explicit implied bounds |
| get added to the <code>fn predicates_of</code> of the relevant item while implicit ones are |
| handled... well... implicitly.</p> |
| <h2 id="explicit-implied-bounds"><a class="header" href="#explicit-implied-bounds">explicit implied bounds</a></h2> |
| <p>The explicit implied bounds are computed in <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_hir_analysis/src/outlives/mod.rs#L20"><code>fn inferred_outlives_of</code></a>. Only ADTs and |
| lazy type aliases have explicit implied bounds which are computed via a fixpoint algorithm |
| in the <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_hir_analysis/src/outlives/mod.rs#L83"><code>fn inferred_outlives_crate</code></a> query.</p> |
| <p>We use <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs#L89"><code>fn insert_required_predicates_to_be_wf</code></a> on all fields of all ADTs in the crate. |
| This function computes the outlives bounds for each component of the field using a |
| separate implementation.</p> |
| <p>For ADTs, trait objects, and associated types the initially required predicates are |
| computed in <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs#L238"><code>fn check_explicit_predicates</code></a>. This simply uses <code>fn explicit_predicates_of</code> |
| without elaborating them.</p> |
| <p>Region predicates are added via <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_hir_analysis/src/outlives/utils.rs#L15"><code>fn insert_outlives_predicate</code></a>. This function takes |
| an outlives predicate, decomposes it and adds the components as explicit predicates only |
| if the outlived region is a region parameter. <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_hir_analysis/src/outlives/utils.rs#L159-L165">It does not add <code>'static</code> requirements</a>.</p> |
| <h2 id="implicit-implied-bounds"><a class="header" href="#implicit-implied-bounds">implicit implied bounds</a></h2> |
| <p>As we are unable to handle implications in binders yet, we cannot simply add the outlives |
| requirements of impls and functions as explicit predicates.</p> |
| <h3 id="using-implicit-implied-bounds-as-assumptions"><a class="header" href="#using-implicit-implied-bounds-as-assumptions">using implicit implied bounds as assumptions</a></h3> |
| <p>These bounds are not added to the <code>ParamEnv</code> of the affected item itself. For lexical |
| region resolution they are added using <a href="https://github.com/rust-lang/rust/blob/8239a37f9c0951a037cfc51763ea52a20e71e6bd/compiler/rustc_infer/src/infer/outlives/env.rs#L50-L55"><code>fn OutlivesEnvironment::from_normalized_bounds</code></a>. |
| Similarly, during MIR borrowck we add them using |
| <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs#L316"><code>fn UniversalRegionRelationsBuilder::add_implied_bounds</code></a>.</p> |
| <p><a href="https://github.com/rust-lang/rust/blob/91cae1dcdcf1a31bd8a92e4a63793d65cfe289bb/compiler/rustc_borrowck/src/type_check/free_region_relations.rs#L258-L332">We add implied bounds for the function signature and impl header in MIR borrowck</a>. |
| Outside of MIR borrowck we add the outlives requirements for the types returned by the |
| <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_ty_utils/src/implied_bounds.rs#L21"><code>fn assumed_wf_types</code></a> query.</p> |
| <p>The assumed outlives constraints for implicit bounds are computed using the |
| <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_traits/src/implied_outlives_bounds.rs#L18C4-L18C27"><code>fn implied_outlives_bounds</code></a> query. This directly |
| <a href="https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs#L95-L96">extracts the required outlives bounds from <code>fn wf::obligations</code></a>.</p> |
| <p>MIR borrowck adds the outlives constraints for both the normalized and unnormalized types, |
| lexical region resolution <a href="https://github.com/rust-lang/rust/blob/91cae1dcdcf1a31bd8a92e4a63793d65cfe289bb/compiler/rustc_trait_selection/src/traits/engine.rs#L227-L250">only uses the unnormalized types</a>.</p> |
| <h3 id="proving-implicit-implied-bounds"><a class="header" href="#proving-implicit-implied-bounds">proving implicit implied bounds</a></h3> |
| <p>As the implicit implied bounds are not included in <code>fn predicates_of</code> we have to |
| separately make sure they actually hold. We generally handle this by checking that |
| all used types are well formed by emitting <code>WellFormed</code> predicates.</p> |
| <p>We cannot emit <code>WellFormed</code> predicates when instantiating impls, as this would result |
| in - currently often inductive - trait solver cycles. We also do not emit constraints |
| involving higher ranked regions as we're lacking the implied bounds from their binder.</p> |
| <p>This results in multiple unsoundnesses:</p> |
| <ul> |
| <li>by using subtyping: <a href="https://github.com/rust-lang/rust/issues/25860">#25860</a></li> |
| <li>by using super trait upcasting for a higher ranked trait bound: <a href="https://github.com/rust-lang/rust/issues/84591">#84591</a></li> |
| <li>by being able to normalize a projection when using an impl while not being able |
| to normalize it when checking the impl: <a href="https://github.com/rust-lang/rust/issues/100051">#100051</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="specialization-1"><a class="header" href="#specialization-1">Specialization</a></h1> |
| <p><strong>TODO</strong>: where does Chalk fit in? Should we mention/discuss it here?</p> |
| <p>Defined in the <code>specialize</code> module.</p> |
| <p>The basic strategy is to build up a <em>specialization graph</em> during |
| coherence checking (coherence checking looks for <a href="traits/../coherence.html">overlapping impls</a>). |
| Insertion into the graph locates the right place |
| to put an impl in the specialization hierarchy; if there is no right |
| place (due to partial overlap but no containment), you get an overlap |
| error. Specialization is consulted when selecting an impl (of course), |
| and the graph is consulted when propagating defaults down the |
| specialization hierarchy.</p> |
| <p>You might expect that the specialization graph would be used during |
| selection – i.e. when actually performing specialization. This is |
| not done for two reasons:</p> |
| <ul> |
| <li> |
| <p>It's merely an optimization: given a set of candidates that apply, |
| we can determine the most specialized one by comparing them directly |
| for specialization, rather than consulting the graph. Given that we |
| also cache the results of selection, the benefit of this |
| optimization is questionable.</p> |
| </li> |
| <li> |
| <p>To build the specialization graph in the first place, we need to use |
| selection (because we need to determine whether one impl specializes |
| another). Dealing with this reentrancy would require some additional |
| mode switch for selection. Given that there seems to be no strong |
| reason to use the graph anyway, we stick with a simpler approach in |
| selection, and use the graph only for propagating default |
| implementations.</p> |
| </li> |
| </ul> |
| <p>Trait impl selection can succeed even when multiple impls can apply, |
| as long as they are part of the same specialization family. In that |
| case, it returns a <em>single</em> impl on success – this is the most |
| specialized impl <em>known</em> to apply. However, if there are any inference |
| variables in play, the returned impl may not be the actual impl we |
| will use at codegen time. Thus, we take special care to avoid projecting |
| associated types unless either (1) the associated type does not use |
| <code>default</code> and thus cannot be overridden or (2) all input types are |
| known concretely.</p> |
| <h2 id="additional-resources-1"><a class="header" href="#additional-resources-1">Additional Resources</a></h2> |
| <p><a href="https://www.youtube.com/watch?v=rZqS4bLPL24">This talk</a> by @sunjay may be useful. Keep in mind that the talk only |
| gives a broad overview of the problem and the solution (it was presented about |
| halfway through @sunjay's work). Also, it was given in June 2018, and some |
| things may have changed by the time you watch it.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="chalk-based-trait-solving"><a class="header" href="#chalk-based-trait-solving">Chalk-based trait solving</a></h1> |
| <p><a href="https://github.com/rust-lang/chalk">Chalk</a> is an experimental trait solver for Rust that is |
| (as of <!-- date-check --> May 2022) under development by the <a href="https://github.com/rust-lang/types-team">Types team</a>. |
| Its goal is to enable a lot of trait system features and bug fixes |
| that are hard to implement (e.g. GATs or specialization). If you would like to |
| help in hacking on the new solver, drop by on the rust-lang Zulip in the <a href="https://rust-lang.zulipchat.com/#narrow/stream/144729-t-types"><code>#t-types</code></a> |
| channel and say hello!</p> |
| <p>The new-style trait solver is based on the work done in <a href="https://github.com/rust-lang/chalk">chalk</a>. Chalk |
| recasts Rust's trait system explicitly in terms of logic programming. It does |
| this by "lowering" Rust code into a kind of logic program we can then execute |
| queries against.</p> |
| <p>The key observation here is that the Rust trait system is basically a |
| kind of logic, and it can be mapped onto standard logical inference |
| rules. We can then look for solutions to those inference rules in a |
| very similar fashion to how e.g. a <a href="https://en.wikipedia.org/wiki/Prolog">Prolog</a> solver works. It turns out |
| that we can't <em>quite</em> use Prolog rules (also called Horn clauses) but |
| rather need a somewhat more expressive variant.</p> |
| <p>You can read more about chalk itself in the |
| <a href="https://rust-lang.github.io/chalk/book/">Chalk book</a> section.</p> |
| <h2 id="ongoing-work"><a class="header" href="#ongoing-work">Ongoing work</a></h2> |
| <p>The design of the new-style trait solving happens in two places:</p> |
| <p><strong>chalk</strong>. The <a href="https://github.com/rust-lang/chalk">chalk</a> repository is where we experiment with new ideas |
| and designs for the trait system.</p> |
| <p><strong>rustc</strong>. Once we are happy with the logical rules, we proceed to |
| implementing them in rustc. We map our struct, trait, and impl declarations |
| into logical inference rules in the lowering module in rustc.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="lowering-to-logic"><a class="header" href="#lowering-to-logic">Lowering to logic</a></h1> |
| <p>The key observation here is that the Rust trait system is basically a |
| kind of logic, and it can be mapped onto standard logical inference |
| rules. We can then look for solutions to those inference rules in a |
| very similar fashion to how e.g. a <a href="https://en.wikipedia.org/wiki/Prolog">Prolog</a> solver works. It turns out |
| that we can't <em>quite</em> use Prolog rules (also called Horn clauses) but |
| rather need a somewhat more expressive variant.</p> |
| <h2 id="rust-traits-and-logic"><a class="header" href="#rust-traits-and-logic">Rust traits and logic</a></h2> |
| <p>One of the first observations is that the Rust trait system is |
| basically a kind of logic. As such, we can map our struct, trait, and |
| impl declarations into logical inference rules. For the most part, |
| these are basically Horn clauses, though we'll see that to capture the |
| full richness of Rust – and in particular to support generic |
| programming – we have to go a bit further than standard Horn clauses.</p> |
| <p>To see how this mapping works, let's start with an example. Imagine |
| we declare a trait and a few impls, like so:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Clone { } |
| impl Clone for usize { } |
| impl<T> Clone for Vec<T> where T: Clone { } |
| <span class="boring">}</span></code></pre></pre> |
| <p>We could map these declarations to some Horn clauses, written in a |
| Prolog-like notation, as follows:</p> |
| <pre><code class="language-text">Clone(usize). |
| Clone(Vec<?T>) :- Clone(?T). |
| |
| // The notation `A :- B` means "A is true if B is true". |
| // Or, put another way, B implies A. |
| </code></pre> |
| <p>In Prolog terms, we might say that <code>Clone(Foo)</code> – where <code>Foo</code> is some |
| Rust type – is a <em>predicate</em> that represents the idea that the type |
| <code>Foo</code> implements <code>Clone</code>. These rules are <strong>program clauses</strong>; they |
| state the conditions under which that predicate can be proven (i.e., |
| considered true). So the first rule just says "Clone is implemented |
| for <code>usize</code>". The next rule says "for any type <code>?T</code>, Clone is |
| implemented for <code>Vec<?T></code> if clone is implemented for <code>?T</code>". So |
| e.g. if we wanted to prove that <code>Clone(Vec<Vec<usize>>)</code>, we would do |
| so by applying the rules recursively:</p> |
| <ul> |
| <li><code>Clone(Vec<Vec<usize>>)</code> is provable if: |
| <ul> |
| <li><code>Clone(Vec<usize>)</code> is provable if: |
| <ul> |
| <li><code>Clone(usize)</code> is provable. (Which it is, so we're all good.)</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p>But now suppose we tried to prove that <code>Clone(Vec<Bar>)</code>. This would |
| fail (after all, I didn't give an impl of <code>Clone</code> for <code>Bar</code>):</p> |
| <ul> |
| <li><code>Clone(Vec<Bar>)</code> is provable if: |
| <ul> |
| <li><code>Clone(Bar)</code> is provable. (But it is not, as there are no applicable rules.)</li> |
| </ul> |
| </li> |
| </ul> |
| <p>We can easily extend the example above to cover generic traits with |
| more than one input type. So imagine the <code>Eq<T></code> trait, which declares |
| that <code>Self</code> is equatable with a value of type <code>T</code>:</p> |
| <pre><code class="language-rust ignore">trait Eq<T> { ... } |
| impl Eq<usize> for usize { } |
| impl<T: Eq<U>> Eq<Vec<U>> for Vec<T> { }</code></pre> |
| <p>That could be mapped as follows:</p> |
| <pre><code class="language-text">Eq(usize, usize). |
| Eq(Vec<?T>, Vec<?U>) :- Eq(?T, ?U). |
| </code></pre> |
| <p>So far so good.</p> |
| <h2 id="type-checking-normal-functions"><a class="header" href="#type-checking-normal-functions">Type-checking normal functions</a></h2> |
| <p>OK, now that we have defined some logical rules that are able to |
| express when traits are implemented and to handle associated types, |
| let's turn our focus a bit towards <strong>type-checking</strong>. Type-checking is |
| interesting because it is what gives us the goals that we need to |
| prove. That is, everything we've seen so far has been about how we |
| derive the rules by which we can prove goals from the traits and impls |
| in the program; but we are also interested in how to derive the goals |
| that we need to prove, and those come from type-checking.</p> |
| <p>Consider type-checking the function <code>foo()</code> here:</p> |
| <pre><code class="language-rust ignore">fn foo() { bar::<usize>() } |
| fn bar<U: Eq<U>>() { }</code></pre> |
| <p>This function is very simple, of course: all it does is to call |
| <code>bar::<usize>()</code>. Now, looking at the definition of <code>bar()</code>, we can see |
| that it has one where-clause <code>U: Eq<U></code>. So, that means that <code>foo()</code> will |
| have to prove that <code>usize: Eq<usize></code> in order to show that it can call <code>bar()</code> |
| with <code>usize</code> as the type argument.</p> |
| <p>If we wanted, we could write a Prolog predicate that defines the |
| conditions under which <code>bar()</code> can be called. We'll say that those |
| conditions are called being "well-formed":</p> |
| <pre><code class="language-text">barWellFormed(?U) :- Eq(?U, ?U). |
| </code></pre> |
| <p>Then we can say that <code>foo()</code> type-checks if the reference to |
| <code>bar::<usize></code> (that is, <code>bar()</code> applied to the type <code>usize</code>) is |
| well-formed:</p> |
| <pre><code class="language-text">fooTypeChecks :- barWellFormed(usize). |
| </code></pre> |
| <p>If we try to prove the goal <code>fooTypeChecks</code>, it will succeed:</p> |
| <ul> |
| <li><code>fooTypeChecks</code> is provable if: |
| <ul> |
| <li><code>barWellFormed(usize)</code>, which is provable if: |
| <ul> |
| <li><code>Eq(usize, usize)</code>, which is provable because of an impl.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p>Ok, so far so good. Let's move on to type-checking a more complex function.</p> |
| <h2 id="type-checking-generic-functions-beyond-horn-clauses"><a class="header" href="#type-checking-generic-functions-beyond-horn-clauses">Type-checking generic functions: beyond Horn clauses</a></h2> |
| <p>In the last section, we used standard Prolog horn-clauses (augmented with Rust's |
| notion of type equality) to type-check some simple Rust functions. But that only |
| works when we are type-checking non-generic functions. If we want to type-check |
| a generic function, it turns out we need a stronger notion of goal than what Prolog |
| can provide. To see what I'm talking about, let's revamp our previous |
| example to make <code>foo</code> generic:</p> |
| <pre><code class="language-rust ignore">fn foo<T: Eq<T>>() { bar::<T>() } |
| fn bar<U: Eq<U>>() { }</code></pre> |
| <p>To type-check the body of <code>foo</code>, we need to be able to hold the type |
| <code>T</code> "abstract". That is, we need to check that the body of <code>foo</code> is |
| type-safe <em>for all types <code>T</code></em>, not just for some specific type. We might express |
| this like so:</p> |
| <pre><code class="language-text">fooTypeChecks :- |
| // for all types T... |
| forall<T> { |
| // ...if we assume that Eq(T, T) is provable... |
| if (Eq(T, T)) { |
| // ...then we can prove that `barWellFormed(T)` holds. |
| barWellFormed(T) |
| } |
| }. |
| </code></pre> |
| <p>This notation I'm using here is the notation I've been using in my |
| prototype implementation; it's similar to standard mathematical |
| notation but a bit Rustified. Anyway, the problem is that standard |
| Horn clauses don't allow universal quantification (<code>forall</code>) or |
| implication (<code>if</code>) in goals (though many Prolog engines do support |
| them, as an extension). For this reason, we need to accept something |
| called "first-order hereditary harrop" (FOHH) clauses – this long |
| name basically means "standard Horn clauses with <code>forall</code> and <code>if</code> in |
| the body". But it's nice to know the proper name, because there is a |
| lot of work describing how to efficiently handle FOHH clauses; see for |
| example Gopalan Nadathur's excellent |
| <a href="https://rust-lang.github.io/chalk/book/bibliography.html#pphhf">"A Proof Procedure for the Logic of Hereditary Harrop Formulas"</a> |
| in <a href="https://rust-lang.github.io/chalk/book/bibliography.html">the bibliography of Chalk Book</a>.</p> |
| <p>It turns out that supporting FOHH is not really all that hard. And |
| once we are able to do that, we can easily describe the type-checking |
| rule for generic functions like <code>foo</code> in our logic.</p> |
| <h2 id="source"><a class="header" href="#source">Source</a></h2> |
| <p>This page is a lightly adapted version of a |
| <a href="http://smallcultfollowing.com/babysteps/blog/2017/01/26/lowering-rust-traits-to-logic/">blog post by Nicholas Matsakis</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="goals-and-clauses"><a class="header" href="#goals-and-clauses">Goals and clauses</a></h1> |
| <p>In logic programming terms, a <strong>goal</strong> is something that you must |
| prove and a <strong>clause</strong> is something that you know is true. As |
| described in the <a href="traits/./lowering-to-logic.html">lowering to logic</a> |
| chapter, Rust's trait solver is based on an extension of hereditary |
| harrop (HH) clauses, which extend traditional Prolog Horn clauses with |
| a few new superpowers.</p> |
| <h2 id="goals-and-clauses-meta-structure"><a class="header" href="#goals-and-clauses-meta-structure">Goals and clauses meta structure</a></h2> |
| <p>In Rust's solver, <strong>goals</strong> and <strong>clauses</strong> have the following forms |
| (note that the two definitions reference one another):</p> |
| <pre><code class="language-text">Goal = DomainGoal // defined in the section below |
| | Goal && Goal |
| | Goal || Goal |
| | exists<K> { Goal } // existential quantification |
| | forall<K> { Goal } // universal quantification |
| | if (Clause) { Goal } // implication |
| | true // something that's trivially true |
| | ambiguous // something that's never provable |
| |
| Clause = DomainGoal |
| | Clause :- Goal // if can prove Goal, then Clause is true |
| | Clause && Clause |
| | forall<K> { Clause } |
| |
| K = <type> // a "kind" |
| | <lifetime> |
| </code></pre> |
| <p>The proof procedure for these sorts of goals is actually quite |
| straightforward. Essentially, it's a form of depth-first search. The |
| paper |
| <a href="https://rust-lang.github.io/chalk/book/bibliography.html#pphhf">"A Proof Procedure for the Logic of Hereditary Harrop Formulas"</a> |
| gives the details.</p> |
| <p>In terms of code, these types are defined in |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_middle/src/traits/mod.rs"><code>rustc_middle/src/traits/mod.rs</code></a> in rustc, and in |
| <a href="https://github.com/rust-lang/chalk/blob/master/chalk-ir/src/lib.rs"><code>chalk-ir/src/lib.rs</code></a> in chalk.</p> |
| <p><a id="domain-goals"></a></p> |
| <h2 id="domain-goals"><a class="header" href="#domain-goals">Domain goals</a></h2> |
| <p><em>Domain goals</em> are the atoms of the trait logic. As can be seen in the |
| definitions given above, general goals basically consist in a combination of |
| domain goals.</p> |
| <p>Moreover, flattening a bit the definition of clauses given previously, one can |
| see that clauses are always of the form:</p> |
| <pre><code class="language-text">forall<K1, ..., Kn> { DomainGoal :- Goal } |
| </code></pre> |
| <p>hence domain goals are in fact clauses' LHS. That is, at the most granular level, |
| domain goals are what the trait solver will end up trying to prove.</p> |
| <p><a id="trait-ref"></a></p> |
| <p>To define the set of domain goals in our system, we need to first |
| introduce a few simple formulations. A <strong>trait reference</strong> consists of |
| the name of a trait along with a suitable set of inputs P0..Pn:</p> |
| <pre><code class="language-text">TraitRef = P0: TraitName<P1..Pn> |
| </code></pre> |
| <p>So, for example, <code>u32: Display</code> is a trait reference, as is <code>Vec<T>: IntoIterator</code>. Note that Rust surface syntax also permits some extra |
| things, like associated type bindings (<code>Vec<T>: IntoIterator<Item = T></code>), that are not part of a trait reference.</p> |
| <p><a id="projection"></a></p> |
| <p>A <strong>projection</strong> consists of an associated item reference along with |
| its inputs P0..Pm:</p> |
| <pre><code class="language-text">Projection = <P0 as TraitName<P1..Pn>>::AssocItem<Pn+1..Pm> |
| </code></pre> |
| <p>Given these, we can define a <code>DomainGoal</code> as follows:</p> |
| <pre><code class="language-text">DomainGoal = Holds(WhereClause) |
| | FromEnv(TraitRef) |
| | FromEnv(Type) |
| | WellFormed(TraitRef) |
| | WellFormed(Type) |
| | Normalize(Projection -> Type) |
| |
| WhereClause = Implemented(TraitRef) |
| | ProjectionEq(Projection = Type) |
| | Outlives(Type: Region) |
| | Outlives(Region: Region) |
| </code></pre> |
| <p><code>WhereClause</code> refers to a <code>where</code> clause that a Rust user would actually be able |
| to write in a Rust program. This abstraction exists only as a convenience as we |
| sometimes want to only deal with domain goals that are effectively writable in |
| Rust.</p> |
| <p>Let's break down each one of these, one-by-one.</p> |
| <h4 id="implementedtraitref"><a class="header" href="#implementedtraitref">Implemented(TraitRef)</a></h4> |
| <p>e.g. <code>Implemented(i32: Copy)</code></p> |
| <p>True if the given trait is implemented for the given input types and lifetimes.</p> |
| <h4 id="projectioneqprojection--type"><a class="header" href="#projectioneqprojection--type">ProjectionEq(Projection = Type)</a></h4> |
| <p>e.g. <code>ProjectionEq<T as Iterator>::Item = u8</code></p> |
| <p>The given associated type <code>Projection</code> is equal to <code>Type</code>; this can be proved |
| with either normalization or using placeholder associated types. See |
| <a href="https://rust-lang.github.io/chalk/book/clauses/type_equality.html">the section on associated types in Chalk Book</a>.</p> |
| <h4 id="normalizeprojection---type"><a class="header" href="#normalizeprojection---type">Normalize(Projection -> Type)</a></h4> |
| <p>e.g. <code>ProjectionEq<T as Iterator>::Item -> u8</code></p> |
| <p>The given associated type <code>Projection</code> can be <a href="https://rust-lang.github.io/chalk/book/clauses/type_equality.html#normalize">normalized</a> to <code>Type</code>.</p> |
| <p>As discussed in <a href="https://rust-lang.github.io/chalk/book/clauses/type_equality.html">the section on associated |
| types in Chalk Book</a>, <code>Normalize</code> implies <code>ProjectionEq</code>, |
| but not vice versa. In general, proving <code>Normalize(<T as Trait>::Item -> U)</code> |
| also requires proving <code>Implemented(T: Trait)</code>.</p> |
| <h4 id="fromenvtraitref"><a class="header" href="#fromenvtraitref">FromEnv(TraitRef)</a></h4> |
| <p>e.g. <code>FromEnv(Self: Add<i32>)</code></p> |
| <p>True if the inner <code>TraitRef</code> is <em>assumed</em> to be true, |
| that is, if it can be derived from the in-scope where clauses.</p> |
| <p>For example, given the following function:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn loud_clone<T: Clone>(stuff: &T) -> T { |
| println!("cloning!"); |
| stuff.clone() |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Inside the body of our function, we would have <code>FromEnv(T: Clone)</code>. In-scope |
| where clauses nest, so a function body inside an impl body inherits the |
| impl body's where clauses, too.</p> |
| <p>This and the next rule are used to implement <a href="https://rust-lang.github.io/chalk/book/clauses/implied_bounds.html#implied-bounds">implied bounds</a>. As we'll see |
| in the section on lowering, <code>FromEnv(TraitRef)</code> implies <code>Implemented(TraitRef)</code>, |
| but not vice versa. This distinction is crucial to implied bounds.</p> |
| <h4 id="fromenvtype"><a class="header" href="#fromenvtype">FromEnv(Type)</a></h4> |
| <p>e.g. <code>FromEnv(HashSet<K>)</code></p> |
| <p>True if the inner <code>Type</code> is <em>assumed</em> to be well-formed, that is, if it is an |
| input type of a function or an impl.</p> |
| <p>For example, given the following code:</p> |
| <pre><code class="language-rust ignore">struct HashSet<K> where K: Hash { ... } |
| |
| fn loud_insert<K>(set: &mut HashSet<K>, item: K) { |
| println!("inserting!"); |
| set.insert(item); |
| }</code></pre> |
| <p><code>HashSet<K></code> is an input type of the <code>loud_insert</code> function. Hence, we assume it |
| to be well-formed, so we would have <code>FromEnv(HashSet<K>)</code> inside the body of our |
| function. As we'll see in the section on lowering, <code>FromEnv(HashSet<K>)</code> implies |
| <code>Implemented(K: Hash)</code> because the |
| <code>HashSet</code> declaration was written with a <code>K: Hash</code> where clause. Hence, we don't |
| need to repeat that bound on the <code>loud_insert</code> function: we rather automatically |
| assume that it is true.</p> |
| <h4 id="wellformeditem"><a class="header" href="#wellformeditem">WellFormed(Item)</a></h4> |
| <p>These goals imply that the given item is <em>well-formed</em>.</p> |
| <p>We can talk about different types of items being well-formed:</p> |
| <ul> |
| <li> |
| <p><em>Types</em>, like <code>WellFormed(Vec<i32>)</code>, which is true in Rust, or |
| <code>WellFormed(Vec<str>)</code>, which is not (because <code>str</code> is not <code>Sized</code>.)</p> |
| </li> |
| <li> |
| <p><em>TraitRefs</em>, like <code>WellFormed(Vec<i32>: Clone)</code>.</p> |
| </li> |
| </ul> |
| <p>Well-formedness is important to <a href="https://rust-lang.github.io/chalk/book/clauses/implied_bounds.html#implied-bounds">implied bounds</a>. In particular, the reason |
| it is okay to assume <code>FromEnv(T: Clone)</code> in the <code>loud_clone</code> example is that we |
| <em>also</em> verify <code>WellFormed(T: Clone)</code> for each call site of <code>loud_clone</code>. |
| Similarly, it is okay to assume <code>FromEnv(HashSet<K>)</code> in the <code>loud_insert</code> |
| example because we will verify <code>WellFormed(HashSet<K>)</code> for each call site of |
| <code>loud_insert</code>.</p> |
| <h4 id="outlivestype-region-outlivesregion-region"><a class="header" href="#outlivestype-region-outlivesregion-region">Outlives(Type: Region), Outlives(Region: Region)</a></h4> |
| <p>e.g. <code>Outlives(&'a str: 'b)</code>, <code>Outlives('a: 'static)</code></p> |
| <p>True if the given type or region on the left outlives the right-hand region.</p> |
| <p><a id="coinductive"></a></p> |
| <h2 id="coinductive-goals"><a class="header" href="#coinductive-goals">Coinductive goals</a></h2> |
| <p>Most goals in our system are "inductive". In an inductive goal, |
| circular reasoning is disallowed. Consider this example clause:</p> |
| <pre><code class="language-text"> Implemented(Foo: Bar) :- |
| Implemented(Foo: Bar). |
| </code></pre> |
| <p>Considered inductively, this clause is useless: if we are trying to |
| prove <code>Implemented(Foo: Bar)</code>, we would then recursively have to prove |
| <code>Implemented(Foo: Bar)</code>, and that cycle would continue ad infinitum |
| (the trait solver will terminate here, it would just consider that |
| <code>Implemented(Foo: Bar)</code> is not known to be true).</p> |
| <p>However, some goals are <em>co-inductive</em>. Simply put, this means that |
| cycles are OK. So, if <code>Bar</code> were a co-inductive trait, then the rule |
| above would be perfectly valid, and it would indicate that |
| <code>Implemented(Foo: Bar)</code> is true.</p> |
| <p><em>Auto traits</em> are one example in Rust where co-inductive goals are used. |
| Consider the <code>Send</code> trait, and imagine that we have this struct:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct Foo { |
| next: Option<Box<Foo>> |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>The default rules for auto traits say that <code>Foo</code> is <code>Send</code> if the |
| types of its fields are <code>Send</code>. Therefore, we would have a rule like</p> |
| <pre><code class="language-text">Implemented(Foo: Send) :- |
| Implemented(Option<Box<Foo>>: Send). |
| </code></pre> |
| <p>As you can probably imagine, proving that <code>Option<Box<Foo>>: Send</code> is |
| going to wind up circularly requiring us to prove that <code>Foo: Send</code> |
| again. So this would be an example where we wind up in a cycle – but |
| that's ok, we <em>do</em> consider <code>Foo: Send</code> to hold, even though it |
| references itself.</p> |
| <p>In general, co-inductive traits are used in Rust trait solving when we |
| want to enumerate a fixed set of possibilities. In the case of auto |
| traits, we are enumerating the set of reachable types from a given |
| starting point (i.e., <code>Foo</code> can reach values of type |
| <code>Option<Box<Foo>></code>, which implies it can reach values of type |
| <code>Box<Foo></code>, and then of type <code>Foo</code>, and then the cycle is complete).</p> |
| <p>In addition to auto traits, <code>WellFormed</code> predicates are co-inductive. |
| These are used to achieve a similar "enumerate all the cases" pattern, |
| as described in the section on <a href="https://rust-lang.github.io/chalk/book/clauses/implied_bounds.html#implied-bounds">implied bounds</a>.</p> |
| <h2 id="incomplete-chapter"><a class="header" href="#incomplete-chapter">Incomplete chapter</a></h2> |
| <p>Some topics yet to be written:</p> |
| <ul> |
| <li>Elaborate on the proof procedure</li> |
| <li>SLG solving – introduce negative reasoning</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="canonical-queries"><a class="header" href="#canonical-queries">Canonical queries</a></h1> |
| <p>The "start" of the trait system is the <strong>canonical query</strong> (these are |
| both queries in the more general sense of the word – something you |
| would like to know the answer to – and in the |
| <a href="traits/../query.html">rustc-specific sense</a>). The idea is that the type |
| checker or other parts of the system, may in the course of doing their |
| thing want to know whether some trait is implemented for some type |
| (e.g., is <code>u32: Debug</code> true?). Or they may want to |
| normalize some associated type.</p> |
| <p>This section covers queries at a fairly high level of abstraction. The |
| subsections look a bit more closely at how these ideas are implemented |
| in rustc.</p> |
| <h2 id="the-traditional-interactive-prolog-query"><a class="header" href="#the-traditional-interactive-prolog-query">The traditional, interactive Prolog query</a></h2> |
| <p>In a traditional Prolog system, when you start a query, the solver |
| will run off and start supplying you with every possible answer it can |
| find. So given something like this:</p> |
| <pre><code class="language-text">?- Vec<i32>: AsRef<?U> |
| </code></pre> |
| <p>The solver might answer:</p> |
| <pre><code class="language-text">Vec<i32>: AsRef<[i32]> |
| continue? (y/n) |
| </code></pre> |
| <p>This <code>continue</code> bit is interesting. The idea in Prolog is that the |
| solver is finding <strong>all possible</strong> instantiations of your query that |
| are true. In this case, if we instantiate <code>?U = [i32]</code>, then the query |
| is true (note that a traditional Prolog interface does not, directly, |
| tell us a value for <code>?U</code>, but we can infer one by unifying the |
| response with our original query – Rust's solver gives back a |
| substitution instead). If we were to hit <code>y</code>, the solver might then |
| give us another possible answer:</p> |
| <pre><code class="language-text">Vec<i32>: AsRef<Vec<i32>> |
| continue? (y/n) |
| </code></pre> |
| <p>This answer derives from the fact that there is a reflexive impl |
| (<code>impl<T> AsRef<T> for T</code>) for <code>AsRef</code>. If were to hit <code>y</code> again, |
| then we might get back a negative response:</p> |
| <pre><code class="language-text">no |
| </code></pre> |
| <p>Naturally, in some cases, there may be no possible answers, and hence |
| the solver will just give me back <code>no</code> right away:</p> |
| <pre><code class="language-text">?- Box<i32>: Copy |
| no |
| </code></pre> |
| <p>In some cases, there might be an infinite number of responses. So for |
| example if I gave this query, and I kept hitting <code>y</code>, then the solver |
| would never stop giving me back answers:</p> |
| <pre><code class="language-text">?- Vec<?U>: Clone |
| Vec<i32>: Clone |
| continue? (y/n) |
| Vec<Box<i32>>: Clone |
| continue? (y/n) |
| Vec<Box<Box<i32>>>: Clone |
| continue? (y/n) |
| Vec<Box<Box<Box<i32>>>>: Clone |
| continue? (y/n) |
| </code></pre> |
| <p>As you can imagine, the solver will gleefully keep adding another |
| layer of <code>Box</code> until we ask it to stop, or it runs out of memory.</p> |
| <p>Another interesting thing is that queries might still have variables |
| in them. For example:</p> |
| <pre><code class="language-text">?- Rc<?T>: Clone |
| </code></pre> |
| <p>might produce the answer:</p> |
| <pre><code class="language-text">Rc<?T>: Clone |
| continue? (y/n) |
| </code></pre> |
| <p>After all, <code>Rc<?T></code> is true <strong>no matter what type <code>?T</code> is</strong>.</p> |
| <p><a id="query-response"></a></p> |
| <h2 id="a-trait-query-in-rustc"><a class="header" href="#a-trait-query-in-rustc">A trait query in rustc</a></h2> |
| <p>The trait queries in rustc work somewhat differently. Instead of |
| trying to enumerate <strong>all possible</strong> answers for you, they are looking |
| for an <strong>unambiguous</strong> answer. In particular, when they tell you the |
| value for a type variable, that means that this is the <strong>only possible |
| instantiation</strong> that you could use, given the current set of impls and |
| where-clauses, that would be provable.</p> |
| <p>The response to a trait query in rustc is typically a |
| <code>Result<QueryResult<T>, NoSolution></code> (where the <code>T</code> will vary a bit |
| depending on the query itself). The <code>Err(NoSolution)</code> case indicates |
| that the query was false and had no answers (e.g., <code>Box<i32>: Copy</code>). |
| Otherwise, the <code>QueryResult</code> gives back information about the possible answer(s) |
| we did find. It consists of four parts:</p> |
| <ul> |
| <li><strong>Certainty:</strong> tells you how sure we are of this answer. It can have two |
| values: |
| <ul> |
| <li><code>Proven</code> means that the result is known to be true. |
| <ul> |
| <li>This might be the result for trying to prove <code>Vec<i32>: Clone</code>, |
| say, or <code>Rc<?T>: Clone</code>.</li> |
| </ul> |
| </li> |
| <li><code>Ambiguous</code> means that there were things we could not yet prove to |
| be either true <em>or</em> false, typically because more type information |
| was needed. (We'll see an example shortly.) |
| <ul> |
| <li>This might be the result for trying to prove <code>Vec<?T>: Clone</code>.</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| <li><strong>Var values:</strong> Values for each of the unbound inference variables |
| (like <code>?T</code>) that appeared in your original query. (Remember that in Prolog, |
| we had to infer these.) |
| <ul> |
| <li>As we'll see in the example below, we can get back var values even |
| for <code>Ambiguous</code> cases.</li> |
| </ul> |
| </li> |
| <li><strong>Region constraints:</strong> these are relations that must hold between |
| the lifetimes that you supplied as inputs. We'll ignore these here.</li> |
| <li><strong>Value:</strong> The query result also comes with a value of type <code>T</code>. For |
| some specialized queries – like normalizing associated types – |
| this is used to carry back an extra result, but it's often just |
| <code>()</code>.</li> |
| </ul> |
| <h3 id="examples-1"><a class="header" href="#examples-1">Examples</a></h3> |
| <p>Let's work through an example query to see what all the parts mean. |
| Consider <a href="https://doc.rust-lang.org/std/borrow/trait.Borrow.html">the <code>Borrow</code> trait</a>. This trait has a number of |
| impls; among them, there are these two (for clarity, I've written the |
| <code>Sized</code> bounds explicitly):</p> |
| <pre><code class="language-rust ignore">impl<T> Borrow<T> for T where T: ?Sized |
| impl<T> Borrow<[T]> for Vec<T> where T: Sized</code></pre> |
| <p><strong>Example 1.</strong> Imagine we are type-checking this (rather artificial) |
| bit of code:</p> |
| <pre><code class="language-rust ignore">fn foo<A, B>(a: A, vec_b: Option<B>) where A: Borrow<B> { } |
| |
| fn main() { |
| let mut t: Vec<_> = vec![]; // Type: Vec<?T> |
| let mut u: Option<_> = None; // Type: Option<?U> |
| foo(t, u); // Example 1: requires `Vec<?T>: Borrow<?U>` |
| ... |
| }</code></pre> |
| <p>As the comments indicate, we first create two variables <code>t</code> and <code>u</code>; |
| <code>t</code> is an empty vector and <code>u</code> is a <code>None</code> option. Both of these |
| variables have unbound inference variables in their type: <code>?T</code> |
| represents the elements in the vector <code>t</code> and <code>?U</code> represents the |
| value stored in the option <code>u</code>. Next, we invoke <code>foo</code>; comparing the |
| signature of <code>foo</code> to its arguments, we wind up with <code>A = Vec<?T></code> and |
| <code>B = ?U</code>. Therefore, the where clause on <code>foo</code> requires that <code>Vec<?T>: Borrow<?U></code>. This is thus our first example trait query.</p> |
| <p>There are many possible solutions to the query <code>Vec<?T>: Borrow<?U></code>; |
| for example:</p> |
| <ul> |
| <li><code>?U = Vec<?T></code>,</li> |
| <li><code>?U = [?T]</code>,</li> |
| <li><code>?T = u32, ?U = [u32]</code></li> |
| <li>and so forth.</li> |
| </ul> |
| <p>Therefore, the result we get back would be as follows (I'm going to |
| ignore region constraints and the "value"):</p> |
| <ul> |
| <li>Certainty: <code>Ambiguous</code> – we're not sure yet if this holds</li> |
| <li>Var values: <code>[?T = ?T, ?U = ?U]</code> – we learned nothing about the values of |
| the variables</li> |
| </ul> |
| <p>In short, the query result says that it is too soon to say much about |
| whether this trait is proven. During type-checking, this is not an |
| immediate error: instead, the type checker would hold on to this |
| requirement (<code>Vec<?T>: Borrow<?U></code>) and wait. As we'll see in the next |
| example, it may happen that <code>?T</code> and <code>?U</code> wind up constrained from |
| other sources, in which case we can try the trait query again.</p> |
| <p><strong>Example 2.</strong> We can now extend our previous example a bit, |
| and assign a value to <code>u</code>:</p> |
| <pre><code class="language-rust ignore">fn foo<A, B>(a: A, vec_b: Option<B>) where A: Borrow<B> { } |
| |
| fn main() { |
| // What we saw before: |
| let mut t: Vec<_> = vec![]; // Type: Vec<?T> |
| let mut u: Option<_> = None; // Type: Option<?U> |
| foo(t, u); // `Vec<?T>: Borrow<?U>` => ambiguous |
| |
| // New stuff: |
| u = Some(vec![]); // ?U = Vec<?V> |
| }</code></pre> |
| <p>As a result of this assignment, the type of <code>u</code> is forced to be |
| <code>Option<Vec<?V>></code>, where <code>?V</code> represents the element type of the |
| vector. This in turn implies that <code>?U</code> is <a href="traits/../type-checking.html">unified</a> to <code>Vec<?V></code>.</p> |
| <p>Let's suppose that the type checker decides to revisit the |
| "as-yet-unproven" trait obligation we saw before, <code>Vec<?T>: Borrow<?U></code>. <code>?U</code> is no longer an unbound inference variable; it now |
| has a value, <code>Vec<?V></code>. So, if we "refresh" the query with that value, we get:</p> |
| <pre><code class="language-text">Vec<?T>: Borrow<Vec<?V>> |
| </code></pre> |
| <p>This time, there is only one impl that applies, the reflexive impl:</p> |
| <pre><code class="language-text">impl<T> Borrow<T> for T where T: ?Sized |
| </code></pre> |
| <p>Therefore, the trait checker will answer:</p> |
| <ul> |
| <li>Certainty: <code>Proven</code></li> |
| <li>Var values: <code>[?T = ?T, ?V = ?T]</code></li> |
| </ul> |
| <p>Here, it is saying that we have indeed proven that the obligation |
| holds, and we also know that <code>?T</code> and <code>?V</code> are the same type (but we |
| don't know what that type is yet!).</p> |
| <p>(In fact, as the function ends here, the type checker would give an |
| error at this point, since the element types of <code>t</code> and <code>u</code> are still |
| not yet known, even though they are known to be the same.)</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="canonicalization"><a class="header" href="#canonicalization">Canonicalization</a></h1> |
| <blockquote> |
| <p><strong>NOTE</strong>: FIXME: The content of this chapter has some overlap with |
| <a href="traits/../solve/canonicalization.html">Next-gen trait solving Canonicalization chapter</a>. |
| It is suggested to reorganize these contents in the future.</p> |
| </blockquote> |
| <p>Canonicalization is the process of <strong>isolating</strong> an inference value |
| from its context. It is a key part of implementing |
| <a href="traits/./canonical-queries.html">canonical queries</a>, and you may wish to read the parent chapter |
| to get more context.</p> |
| <p>Canonicalization is really based on a very simple concept: every |
| <a href="traits/../type-inference.html#vars">inference variable</a> is always in one of |
| two states: either it is <strong>unbound</strong>, in which case we don't know yet |
| what type it is, or it is <strong>bound</strong>, in which case we do. So to |
| isolate some data-structure T that contains types/regions from its |
| environment, we just walk down and find the unbound variables that |
| appear in T; those variables get replaced with "canonical variables", |
| starting from zero and numbered in a fixed order (left to right, for |
| the most part, but really it doesn't matter as long as it is |
| consistent).</p> |
| <p>So, for example, if we have the type <code>X = (?T, ?U)</code>, where <code>?T</code> and |
| <code>?U</code> are distinct, unbound inference variables, then the canonical |
| form of <code>X</code> would be <code>(?0, ?1)</code>, where <code>?0</code> and <code>?1</code> represent these |
| <strong>canonical placeholders</strong>. Note that the type <code>Y = (?U, ?T)</code> also |
| canonicalizes to <code>(?0, ?1)</code>. But the type <code>Z = (?T, ?T)</code> would |
| canonicalize to <code>(?0, ?0)</code> (as would <code>(?U, ?U)</code>). In other words, the |
| exact identity of the inference variables is not important – unless |
| they are repeated.</p> |
| <p>We use this to improve caching as well as to detect cycles and other |
| things during trait resolution. Roughly speaking, the idea is that if |
| two trait queries have the same canonical form, then they will get |
| the same answer. That answer will be expressed in terms of the |
| canonical variables (<code>?0</code>, <code>?1</code>), which we can then map back to the |
| original variables (<code>?T</code>, <code>?U</code>).</p> |
| <h2 id="canonicalizing-the-query"><a class="header" href="#canonicalizing-the-query">Canonicalizing the query</a></h2> |
| <p>To see how it works, imagine that we are asking to solve the following |
| trait query: <code>?A: Foo<'static, ?B></code>, where <code>?A</code> and <code>?B</code> are unbound. |
| This query contains two unbound variables, but it also contains the |
| lifetime <code>'static</code>. The trait system generally ignores all lifetimes |
| and treats them equally, so when canonicalizing, we will <em>also</em> |
| replace any <a href="traits/../appendix/background.html#free-vs-bound">free lifetime</a> with a |
| canonical variable (Note that <code>'static</code> is actually a <em>free</em> lifetime |
| variable here. We are not considering it in the typing context of the whole |
| program but only in the context of this trait reference. Mathematically, we |
| are not quantifying over the whole program, but only this obligation). |
| Therefore, we get the following result:</p> |
| <pre><code class="language-text">?0: Foo<'?1, ?2> |
| </code></pre> |
| <p>Sometimes we write this differently, like so:</p> |
| <pre><code class="language-text">for<T,L,T> { ?0: Foo<'?1, ?2> } |
| </code></pre> |
| <p>This <code>for<></code> gives some information about each of the canonical |
| variables within. In this case, each <code>T</code> indicates a type variable, |
| so <code>?0</code> and <code>?2</code> are types; the <code>L</code> indicates a lifetime variable, so |
| <code>?1</code> is a lifetime. The <code>canonicalize</code> method <em>also</em> gives back a |
| <code>CanonicalVarValues</code> array OV with the "original values" for each |
| canonicalized variable:</p> |
| <pre><code class="language-text">[?A, 'static, ?B] |
| </code></pre> |
| <p>We'll need this vector OV later, when we process the query response.</p> |
| <h2 id="executing-the-query"><a class="header" href="#executing-the-query">Executing the query</a></h2> |
| <p>Once we've constructed the canonical query, we can try to solve it. |
| To do so, we will wind up creating a fresh inference context and |
| <strong>instantiating</strong> the canonical query in that context. The idea is that |
| we create a substitution S from the canonical form containing a fresh |
| inference variable (of suitable kind) for each canonical variable. |
| So, for our example query:</p> |
| <pre><code class="language-text">for<T,L,T> { ?0: Foo<'?1, ?2> } |
| </code></pre> |
| <p>the substitution S might be:</p> |
| <pre><code class="language-text">S = [?A, '?B, ?C] |
| </code></pre> |
| <p>We can then replace the bound canonical variables (<code>?0</code>, etc) with |
| these inference variables, yielding the following fully instantiated |
| query:</p> |
| <pre><code class="language-text">?A: Foo<'?B, ?C> |
| </code></pre> |
| <p>Remember that substitution S though! We're going to need it later.</p> |
| <p>OK, now that we have a fresh inference context and an instantiated |
| query, we can go ahead and try to solve it. The trait solver itself is |
| explained in more detail in <a href="traits/../solve/the-solver.html">another section</a>, but |
| suffice to say that it will compute a <a href="traits/./canonical-queries.html#query-response">certainty value</a> (<code>Proven</code> or |
| <code>Ambiguous</code>) and have side-effects on the inference variables we've |
| created. For example, if there were only one impl of <code>Foo</code>, like so:</p> |
| <pre><code class="language-rust ignore">impl<'a, X> Foo<'a, X> for Vec<X> |
| where X: 'a |
| { ... }</code></pre> |
| <p>then we might wind up with a certainty value of <code>Proven</code>, as well as |
| creating fresh inference variables <code>'?D</code> and <code>?E</code> (to represent the |
| parameters on the impl) and unifying as follows:</p> |
| <ul> |
| <li><code>'?B = '?D</code></li> |
| <li><code>?A = Vec<?E></code></li> |
| <li><code>?C = ?E</code></li> |
| </ul> |
| <p>We would also accumulate the region constraint <code>?E: '?D</code>, due to the |
| where clause.</p> |
| <p>In order to create our final query result, we have to "lift" these |
| values out of the query's inference context and into something that |
| can be reapplied in our original inference context. We do that by |
| <strong>re-applying canonicalization</strong>, but to the <strong>query result</strong>.</p> |
| <h2 id="canonicalizing-the-query-result"><a class="header" href="#canonicalizing-the-query-result">Canonicalizing the query result</a></h2> |
| <p>As discussed in <a href="traits/./canonical-queries.html#query-response">the parent section</a>, most trait queries wind up |
| with a result that brings together a "certainty value" <code>certainty</code>, a |
| result substitution <code>var_values</code>, and some region constraints. To |
| create this, we wind up re-using the substitution S that we created |
| when first instantiating our query. To refresh your memory, we had a query</p> |
| <pre><code class="language-text">for<T,L,T> { ?0: Foo<'?1, ?2> } |
| </code></pre> |
| <p>for which we made a substutition S:</p> |
| <pre><code class="language-text">S = [?A, '?B, ?C] |
| </code></pre> |
| <p>We then did some work which unified some of those variables with other things. |
| If we "refresh" S with the latest results, we get:</p> |
| <pre><code class="language-text">S = [Vec<?E>, '?D, ?E] |
| </code></pre> |
| <p>These are precisely the new values for the three input variables from |
| our original query. Note though that they include some new variables |
| (like <code>?E</code>). We can make those go away by canonicalizing again! We don't |
| just canonicalize S, though, we canonicalize the whole query response QR:</p> |
| <pre><code class="language-text">QR = { |
| certainty: Proven, // or whatever |
| var_values: [Vec<?E>, '?D, ?E] // this is S |
| region_constraints: [?E: '?D], // from the impl |
| value: (), // for our purposes, just (), but |
| // in some cases this might have |
| // a type or other info |
| } |
| </code></pre> |
| <p>The result would be as follows:</p> |
| <pre><code class="language-text">Canonical(QR) = for<T, L> { |
| certainty: Proven, |
| var_values: [Vec<?0>, '?1, ?0] |
| region_constraints: [?0: '?1], |
| value: (), |
| } |
| </code></pre> |
| <p>(One subtle point: when we canonicalize the query <strong>result</strong>, we do not |
| use any special treatment for free lifetimes. Note that both |
| references to <code>'?D</code>, for example, were converted into the same |
| canonical variable (<code>?1</code>). This is in contrast to the original query, |
| where we canonicalized every free lifetime into a fresh canonical |
| variable.)</p> |
| <p>Now, this result must be reapplied in each context where needed.</p> |
| <h2 id="processing-the-canonicalized-query-result"><a class="header" href="#processing-the-canonicalized-query-result">Processing the canonicalized query result</a></h2> |
| <p>In the previous section we produced a canonical query result. We now have |
| to apply that result in our original context. If you recall, way back in the |
| beginning, we were trying to prove this query:</p> |
| <pre><code class="language-text">?A: Foo<'static, ?B> |
| </code></pre> |
| <p>We canonicalized that into this:</p> |
| <pre><code class="language-text">for<T,L,T> { ?0: Foo<'?1, ?2> } |
| </code></pre> |
| <p>and now we got back a canonical response:</p> |
| <pre><code class="language-text">for<T, L> { |
| certainty: Proven, |
| var_values: [Vec<?0>, '?1, ?0] |
| region_constraints: [?0: '?1], |
| value: (), |
| } |
| </code></pre> |
| <p>We now want to apply that response to our context. Conceptually, how |
| we do that is to (a) instantiate each of the canonical variables in |
| the result with a fresh inference variable, (b) unify the values in |
| the result with the original values, and then (c) record the region |
| constraints for later. Doing step (a) would yield a result of</p> |
| <pre><code class="language-text">{ |
| certainty: Proven, |
| var_values: [Vec<?C>, '?D, ?C] |
| ^^ ^^^ fresh inference variables |
| region_constraints: [?C: '?D], |
| value: (), |
| } |
| </code></pre> |
| <p>Step (b) would then unify:</p> |
| <pre><code class="language-text">?A with Vec<?C> |
| 'static with '?D |
| ?B with ?C |
| </code></pre> |
| <p>And finally the region constraint of <code>?C: 'static</code> would be recorded |
| for later verification.</p> |
| <p>(What we <em>actually</em> do is a mildly optimized variant of that: Rather |
| than eagerly instantiating all of the canonical values in the result |
| with variables, we instead walk the vector of values, looking for |
| cases where the value is just a canonical variable. In our example, |
| <code>values[2]</code> is <code>?C</code>, so that means we can deduce that <code>?C := ?B</code> and |
| <code>'?D := 'static</code>. This gives us a partial set of values. Anything for |
| which we do not find a value, we create an inference variable.)</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="trait-solving-new"><a class="header" href="#trait-solving-new">Trait solving (new)</a></h1> |
| <p>This chapter describes how trait solving works with the new WIP solver located in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/solve/index.html"><code>rustc_trait_selection/solve</code></a>. Feel free to also look at the docs for |
| <a href="solve/../traits/resolution.html">the current solver</a> and <a href="solve/../traits/chalk.html">the chalk solver</a>.</p> |
| <h2 id="core-concepts"><a class="header" href="#core-concepts">Core concepts</a></h2> |
| <p>The goal of the trait system is to check whether a given trait bound is satisfied. |
| Most notably when typechecking the body of - potentially generic - functions. |
| For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn uses_vec_clone<T: Clone>(x: Vec<T>) -> (Vec<T>, Vec<T>) { |
| (x.clone(), x) |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Here the call to <code>x.clone()</code> requires us to prove that <code>Vec<T></code> implements <code>Clone</code> given |
| the assumption that <code>T: Clone</code> is true. We can assume <code>T: Clone</code> as that will be proven by |
| callers of this function.</p> |
| <p>The concept of "prove the <code>Vec<T>: Clone</code> with the assumption <code>T: Clone</code>" is called a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/canonical/ir/solve/struct.Goal.html"><code>Goal</code></a>. |
| Both <code>Vec<T>: Clone</code> and <code>T: Clone</code> are represented using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Predicate.html"><code>Predicate</code></a>. There are other |
| predicates, most notably equality bounds on associated items: <code><Vec<T> as IntoIterator>::Item == T</code>. |
| See the <code>PredicateKind</code> enum for an exhaustive list. A <code>Goal</code> is represented as the <code>predicate</code> we |
| have to prove and the <code>param_env</code> in which this predicate has to hold.</p> |
| <p>We prove goals by checking whether each possible <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/assembly/struct.Candidate.html"><code>Candidate</code></a> applies for the given goal by |
| recursively proving its nested goals. For a list of possible candidates with examples, look at |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/canonical/ir/solve/enum.CandidateSource.html"><code>CandidateSource</code></a>. The most important candidates are <code>Impl</code> candidates, i.e. trait implementations |
| written by the user, and <code>ParamEnv</code> candidates, i.e. assumptions in our current environment.</p> |
| <p>Looking at the above example, to prove <code>Vec<T>: Clone</code> we first use |
| <code>impl<T: Clone> Clone for Vec<T></code>. To use this impl we have to prove the nested |
| goal that <code>T: Clone</code> holds. This can use the assumption <code>T: Clone</code> from the <code>ParamEnv</code> |
| which does not have any nested goals. Therefore <code>Vec<T>: Clone</code> holds.</p> |
| <p>The trait solver can either return success, ambiguity or an error as a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/solve/type.CanonicalResponse.html"><code>CanonicalResponse</code></a>. |
| For success and ambiguity it also returns constraints inference and region constraints.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="invariants-of-the-type-system"><a class="header" href="#invariants-of-the-type-system">Invariants of the type system</a></h1> |
| <p>FIXME: This file talks about invariants of the type system as a whole, not only the solver</p> |
| <p>There are a lot of invariants - things the type system guarantees to be true at all times - |
| which are desirable or expected from other languages and type systems. Unfortunately, quite |
| a few of them do not hold in Rust right now. This is either fundamental to its design or |
| caused by bugs, and something that may change in the future.</p> |
| <p>It is important to know about the things you can assume while working on, and with, the |
| type system, so here's an incomplete and unofficial list of invariants of |
| the core type system:</p> |
| <ul> |
| <li>✅: this invariant mostly holds, with some weird exceptions or current bugs</li> |
| <li>❌: this invariant does not hold, and is unlikely to do so in the future; do not rely on it for soundness or have to be incredibly careful when doing so</li> |
| </ul> |
| <h3 id="wfx-implies-wfnormalizex-"><a class="header" href="#wfx-implies-wfnormalizex-"><code>wf(X)</code> implies <code>wf(normalize(X))</code> ✅</a></h3> |
| <p>If a type containing aliases is well-formed, it should also be |
| well-formed after normalizing said aliases. We rely on this as |
| otherwise we would have to re-check for well-formedness for these |
| types.</p> |
| <p>This currently does not hold due to a type system unsoundness: <a href="https://github.com/rust-lang/rust/issues/84533">#84533</a>.</p> |
| <h3 id="structural-equality-modulo-regions-implies-semantic-equality-"><a class="header" href="#structural-equality-modulo-regions-implies-semantic-equality-">Structural equality modulo regions implies semantic equality ✅</a></h3> |
| <p>If you have a some type and equate it to itself after replacing any regions with unique |
| inference variables in both the lhs and rhs, the now potentially structurally different |
| types should still be equal to each other.</p> |
| <p>This is needed to prevent goals from succeeding in HIR typeck and then failing in MIR borrowck. |
| If this invariant is broken, MIR typeck ends up failing with an ICE.</p> |
| <h3 id="applying-inference-results-from-a-goal-does-not-change-its-result-"><a class="header" href="#applying-inference-results-from-a-goal-does-not-change-its-result-">Applying inference results from a goal does not change its result ❌</a></h3> |
| <p>TODO: this invariant is formulated in a weird way and needs to be elaborated. |
| Pretty much: I would like this check to only fail if there's a solver bug: |
| <a href="https://github.com/rust-lang/rust/blob/2ffeb4636b4ae376f716dc4378a7efb37632dc2d/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs#L391-L407">https://github.com/rust-lang/rust/blob/2ffeb4636b4ae376f716dc4378a7efb37632dc2d/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs#L391-L407</a>. |
| We should readd this check and see where it breaks :3</p> |
| <p>If we prove some goal/equate types/whatever, apply the resulting inference constraints, |
| and then redo the original action, the result should be the same.</p> |
| <p>This unfortunately does not hold - at least in the new solver - due to a few annoying reasons.</p> |
| <h3 id="the-trait-solver-has-to-be-locally-sound-"><a class="header" href="#the-trait-solver-has-to-be-locally-sound-">The trait solver has to be <em>locally sound</em> ✅</a></h3> |
| <p>This means that we must never return <em>success</em> for goals for which no <code>impl</code> exists. That would |
| mean we assume a trait is implemented even though it is not, which is very likely to result in |
| actual unsoundness. When using <code>where</code>-bounds to prove a goal, the <code>impl</code> will be provided by the |
| user of the item.</p> |
| <p>This invariant only holds if we check region constraints. As we do not check region constraints |
| during implicit negative overlap check in coherence, this invariant is broken there. As this check |
| relies on <em>completeness</em> of the trait solver, it is not able to use the current region constraints |
| check - <code>InferCtxt::resolve_regions</code> - as its handling of type outlives goals is incomplete.</p> |
| <h3 id="normalization-of-semantically-equal-aliases-in-empty-environments-results-in-a-unique-type-"><a class="header" href="#normalization-of-semantically-equal-aliases-in-empty-environments-results-in-a-unique-type-">Normalization of semantically equal aliases in empty environments results in a unique type ✅</a></h3> |
| <p>Normalization for alias types/consts has to have a unique result. Otherwise we can easily |
| implement transmute in safe code. Given the following function, we have to make sure that |
| the input and output types always get normalized to the same concrete type.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T: Trait>( |
| x: <T as Trait>::Assoc |
| ) -> <T as Trait>::Assoc { |
| x |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Many of the currently known unsound issues end up relying on this invariant being broken. |
| It is however very difficult to imagine a sound type system without this invariant, so |
| the issue is that the invariant is broken, not that we incorrectly rely on it.</p> |
| <h3 id="the-type-system-is-complete-"><a class="header" href="#the-type-system-is-complete-">The type system is complete ❌</a></h3> |
| <p>The type system is not complete. |
| It often adds unnecessary inference constraints, and errors even though the goal could hold.</p> |
| <ul> |
| <li>method selection</li> |
| <li>opaque type inference</li> |
| <li>handling type outlives constraints</li> |
| <li>preferring <code>ParamEnv</code> candidates over <code>Impl</code> candidates during candidate selection |
| in the trait solver</li> |
| </ul> |
| <h3 id="goals-keep-their-result-from-hir-typeck-afterwards-"><a class="header" href="#goals-keep-their-result-from-hir-typeck-afterwards-">Goals keep their result from HIR typeck afterwards ✅</a></h3> |
| <p>Having a goal which succeeds during HIR typeck but fails when being reevaluated during MIR borrowck causes ICE, e.g. <a href="https://github.com/rust-lang/rust/issues/140211">#140211</a>.</p> |
| <p>Having a goal which succeeds during HIR typeck but fails after being instantiated is unsound, e.g. <a href="https://github.com/rust-lang/rust/issues/140212">#140212</a>.</p> |
| <p>It is interesting that we allow some incompleteness in the trait solver while still maintaining this limitation. It would be nice if there was a clear way to separate the "allowed incompleteness" from behavior which would break this invariant.</p> |
| <h4 id="normalization-must-not-change-results"><a class="header" href="#normalization-must-not-change-results">Normalization must not change results</a></h4> |
| <p>This invariant is relied on to allow the normalization of generic aliases. Breaking |
| it can easily result in unsoundness, e.g. <a href="https://github.com/rust-lang/rust/issues/57893">#57893</a>.</p> |
| <h4 id="goals-may-still-overflow-after-instantiation"><a class="header" href="#goals-may-still-overflow-after-instantiation">Goals may still overflow after instantiation</a></h4> |
| <p>This happens they start to hit the recursion limit. |
| We also have diverging aliases which are scuffed. |
| It's unclear how these should be handled :3</p> |
| <h3 id="trait-goals-in-empty-environments-are-proven-by-a-unique-impl-"><a class="header" href="#trait-goals-in-empty-environments-are-proven-by-a-unique-impl-">Trait goals in empty environments are proven by a unique impl ✅</a></h3> |
| <p>If a trait goal holds with an empty environment, there should be a unique <code>impl</code>, |
| either user-defined or builtin, which is used to prove that goal. This is |
| necessary to select unique methods and associated items.</p> |
| <p>We do however break this invariant in a few cases, some of which are due to bugs, some by design:</p> |
| <ul> |
| <li><em>marker traits</em> are allowed to overlap as they do not have associated items</li> |
| <li><em>specialization</em> allows specializing impls to overlap with their parent</li> |
| <li>the builtin trait object trait implementation can overlap with a user-defined impl: |
| <a href="https://github.com/rust-lang/rust/issues/57893">#57893</a></li> |
| </ul> |
| <h4 id="the-type-system-is-complete-during-the-implicit-negative-overlap-check-in-coherence-"><a class="header" href="#the-type-system-is-complete-during-the-implicit-negative-overlap-check-in-coherence-">The type system is complete during the implicit negative overlap check in coherence ✅</a></h4> |
| <p>For more on overlap checking, see <a href="solve/../coherence.html">Coherence chapter</a>.</p> |
| <p>During the implicit negative overlap check in coherence, |
| we must never return <em>error</em> for goals which can be proven. |
| This would allow for overlapping impls with potentially different associated items, |
| breaking a bunch of other invariants.</p> |
| <p>This invariant is currently broken in many different ways while actually something we rely on. |
| We have to be careful as it is quite easy to break:</p> |
| <ul> |
| <li>generalization of aliases</li> |
| <li>generalization during subtyping binders (luckily not exploitable in coherence)</li> |
| </ul> |
| <h3 id="trait-solving-must-not-depend-on-lifetimes-being-different-"><a class="header" href="#trait-solving-must-not-depend-on-lifetimes-being-different-">Trait solving must not depend on lifetimes being different ✅</a></h3> |
| <p>If a goal holds with lifetimes being different, it must also hold with these lifetimes being the same. We otherwise get post-monomorphization errors during codegen or unsoundness due to invalid vtables.</p> |
| <p>We could also just get inconsistent behavior when first proving a goal with different lifetimes which are later constrained to be equal.</p> |
| <h3 id="trait-solving-in-bodies-must-not-depend-on-lifetimes-being-equal-"><a class="header" href="#trait-solving-in-bodies-must-not-depend-on-lifetimes-being-equal-">Trait solving in bodies must not depend on lifetimes being equal ✅</a></h3> |
| <p>We also have to be careful with relying on equality of regions in the trait solver. |
| This is fine for codegen, as we treat all erased regions as equal. We can however |
| lose equality information from HIR to MIR typeck.</p> |
| <p>This currently does not hold with the new solver: <a href="https://github.com/rust-lang/trait-system-refactor-initiative/issues/27">trait-system-refactor-initiative#27</a>.</p> |
| <h3 id="removing-ambiguity-makes-strictly-more-things-compile-"><a class="header" href="#removing-ambiguity-makes-strictly-more-things-compile-">Removing ambiguity makes strictly more things compile ❌</a></h3> |
| <p>Ideally we <em>should</em> not rely on ambiguity for things to compile. |
| Not doing that will cause future improvements to be breaking changes.</p> |
| <p>Due to <em>incompleteness</em> this is not the case, |
| and improving inference can result in inference changes, breaking existing projects.</p> |
| <h3 id="semantic-equality-implies-structural-equality-"><a class="header" href="#semantic-equality-implies-structural-equality-">Semantic equality implies structural equality ✅</a></h3> |
| <p>Two types being equal in the type system must mean that they have the |
| same <code>TypeId</code> after instantiating their generic parameters with concrete |
| arguments. We can otherwise use their different <code>TypeId</code>s to impact trait selection.</p> |
| <p>We lookup types using structural equality during codegen, but this shouldn't necessarily be unsound</p> |
| <ul> |
| <li>may result in redundant method codegen or backend type check errors?</li> |
| <li>we also rely on it in CTFE assertions</li> |
| </ul> |
| <h3 id="semantically-different-types-have-different-typeids-"><a class="header" href="#semantically-different-types-have-different-typeids-">Semantically different types have different <code>TypeId</code>s ✅</a></h3> |
| <p>Semantically different <code>'static</code> types need different <code>TypeId</code>s to avoid transmutes, |
| for example <code>for<'a> fn(&'a str)</code> vs <code>fn(&'static str)</code> must have a different <code>TypeId</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-solver"><a class="header" href="#the-solver">The solver</a></h1> |
| <p>Also consider reading the documentation for <a href="https://rust-lang.github.io/chalk/book/recursive.html">the recursive solver in chalk</a> |
| as it is very similar to this implementation and also talks about limitations of this |
| approach.</p> |
| <h2 id="a-rough-walkthrough"><a class="header" href="#a-rough-walkthrough">A rough walkthrough</a></h2> |
| <p>The entry-point of the solver is <code>InferCtxtEvalExt::evaluate_root_goal</code>. This |
| function sets up the root <code>EvalCtxt</code> and then calls <code>EvalCtxt::evaluate_goal</code>, |
| to actually enter the trait solver.</p> |
| <p><code>EvalCtxt::evaluate_goal</code> handles <a href="solve/./canonicalization.html">canonicalization</a>, caching, |
| overflow, and solver cycles. Once that is done, it creates a nested <code>EvalCtxt</code> with a |
| separate local <code>InferCtxt</code> and calls <code>EvalCtxt::compute_goal</code>, which is responsible for the |
| 'actual solver behavior'. We match on the <code>PredicateKind</code>, delegating to a separate function |
| for each one.</p> |
| <p>For trait goals, such a <code>Vec<T>: Clone</code>, <code>EvalCtxt::compute_trait_goal</code> has |
| to collect all the possible ways this goal can be proven via |
| <code>EvalCtxt::assemble_and_evaluate_candidates</code>. Each candidate is handled in |
| a separate "probe", to not leak inference constraints to the other candidates. |
| We then try to merge the assembled candidates via <code>EvalCtxt::merge_candidates</code>.</p> |
| <h2 id="important-concepts-and-design-patterns"><a class="header" href="#important-concepts-and-design-patterns">Important concepts and design patterns</a></h2> |
| <h3 id="evalctxtadd_goal"><a class="header" href="#evalctxtadd_goal"><code>EvalCtxt::add_goal</code></a></h3> |
| <p>To prove nested goals, we don't directly call <code>EvalCtxt::compute_goal</code>, but instead |
| add the goal to the <code>EvalCtxt</code> with <code>EvalCtxt::all_goal</code>. We then prove all nested |
| goals together in either <code>EvalCtxt::try_evaluate_added_goals</code> or |
| <code>EvalCtxt::evaluate_added_goals_and_make_canonical_response</code>. This allows us to handle |
| inference constraints from later goals.</p> |
| <p>E.g. if we have both <code>?x: Debug</code> and <code>(): ConstrainToU8<?x></code> as nested goals, |
| then proving <code>?x: Debug</code> is initially ambiguous, but after proving <code>(): ConstrainToU8<?x></code> |
| we constrain <code>?x</code> to <code>u8</code> and proving <code>u8: Debug</code> succeeds.</p> |
| <h3 id="matching-on-tykind"><a class="header" href="#matching-on-tykind">Matching on <code>TyKind</code></a></h3> |
| <p>We lazily normalize types in the solver, so we always have to assume that any types |
| and constants are potentially unnormalized. This means that matching on <code>TyKind</code> can easily |
| be incorrect.</p> |
| <p>We handle normalization in two different ways. When proving <code>Trait</code> goals when normalizing |
| associated types, we separately assemble candidates depending on whether they structurally |
| match the self type. Candidates which match on the self type are handled in |
| <code>EvalCtxt::assemble_candidates_via_self_ty</code> which recurses via |
| <code>EvalCtxt::assemble_candidates_after_normalizing_self_ty</code>, which normalizes the self type |
| by one level. In all other cases we have to match on a <code>TyKind</code> we first use |
| <code>EvalCtxt::try_normalize_ty</code> to normalize the type as much as possible.</p> |
| <h3 id="higher-ranked-goals"><a class="header" href="#higher-ranked-goals">Higher ranked goals</a></h3> |
| <p>In case the goal is higher-ranked, e.g. <code>for<'a> F: FnOnce(&'a ())</code>, <code>EvalCtxt::compute_goal</code> |
| eagerly instantiates <code>'a</code> with a placeholder and then recursively proves |
| <code>F: FnOnce(&'!a ())</code> as a nested goal.</p> |
| <h3 id="dealing-with-choice"><a class="header" href="#dealing-with-choice">Dealing with choice</a></h3> |
| <p>Some goals can be proven in multiple ways. In these cases we try each option in |
| a separate "probe" and then attempt to merge the resulting responses by using |
| <code>EvalCtxt::try_merge_responses</code>. If merging the responses fails, we use |
| <code>EvalCtxt::flounder</code> instead, returning ambiguity. For some goals, we try to |
| incompletely prefer some choices over others in case <code>EvalCtxt::try_merge_responses</code> |
| fails.</p> |
| <h2 id="learning-more"><a class="header" href="#learning-more">Learning more</a></h2> |
| <p>The solver should be fairly self-contained. I hope that the above information provides a |
| good foundation when looking at the code itself. Please reach out on Zulip if you get stuck |
| while doing so or there are some quirks and design decisions which were unclear and deserve |
| better comments or should be mentioned here.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="candidate-preference"><a class="header" href="#candidate-preference">Candidate preference</a></h1> |
| <p>There are multiple ways to prove <code>Trait</code> and <code>NormalizesTo</code> goals. Each such option is called a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/assembly/struct.Candidate.html"><code>Candidate</code></a>. If there are multiple applicable candidates, we prefer some candidates over others. We store the relevant information in their <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/enum.CandidateSource.html"><code>CandidateSource</code></a>.</p> |
| <p>This preference may result in incorrect inference or region constraints and would therefore be unsound during coherence. Because of this, we simply try to merge all candidates in coherence.</p> |
| <h2 id="trait-goals"><a class="header" href="#trait-goals"><code>Trait</code> goals</a></h2> |
| <p>Trait goals merge their applicable candidates in <a href="https://github.com/rust-lang/rust/blob/e3ee7f7aea5b45af3b42b5e4713da43876a65ac9/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs#L1342-L1424"><code>fn merge_trait_candidates</code></a>. This document provides additional details and references to explain <em>why</em> we've got the current preference rules.</p> |
| <h3 id="candidatesourcebuiltinimplbuiltinimplsourcetrivial"><a class="header" href="#candidatesourcebuiltinimplbuiltinimplsourcetrivial"><code>CandidateSource::BuiltinImpl(BuiltinImplSource::Trivial))</code></a></h3> |
| <p>Trivial builtin impls are builtin impls which are known to be always applicable for well-formed types. This means that if one exists, using another candidate should never have fewer constraints. We currently only consider <code>Sized</code> - and <code>MetaSized</code> - impls to be trivial.</p> |
| <p>This is necessary to prevent a lifetime error for the following pattern</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<T>: Sized {} |
| impl<'a> Trait<u32> for &'a str {} |
| impl<'a> Trait<i32> for &'a str {} |
| fn is_sized<T: Sized>(_: T) {} |
| fn foo<'a, 'b, T>(x: &'b str) |
| where |
| &'a str: Trait<T>, |
| { |
| // Elaborating the `&'a str: Trait<T>` where-bound results in a |
| // `&'a str: Sized` where-bound. We do not want to prefer this |
| // over the builtin impl. |
| is_sized(x); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This preference is incorrect in case the builtin impl has a nested goal which relies on a non-param where-clause</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct MyType<'a, T: ?Sized>(&'a (), T); |
| fn is_sized<T>() {} |
| fn foo<'a, T: ?Sized>() |
| where |
| (MyType<'a, T>,): Sized, |
| MyType<'static, T>: Sized, |
| { |
| // The where-bound is trivial while the builtin `Sized` impl for tuples |
| // requires proving `MyType<'a, T>: Sized` which can only be proven by |
| // using the where-clause, adding an unnecessary `'static` constraint. |
| is_sized::<(MyType<'a, T>,)>(); |
| //~^ ERROR lifetime may not live long enough |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="candidatesourceparamenv"><a class="header" href="#candidatesourceparamenv"><code>CandidateSource::ParamEnv</code></a></h3> |
| <p>Once there's at least one <em>non-global</em> <code>ParamEnv</code> candidate, we prefer <em>all</em> <code>ParamEnv</code> candidates over other candidate kinds. |
| A where-bound is global if it is not higher-ranked and doesn't contain any generic parameters. It may contain <code>'static</code>.</p> |
| <p>We try to apply where-bounds over other candidates as users tends to have the most control over them, so they can most easily |
| adjust them in case our candidate preference is incorrect.</p> |
| <h4 id="preference-over-impl-candidates"><a class="header" href="#preference-over-impl-candidates">Preference over <code>Impl</code> candidates</a></h4> |
| <p>This is necessary to avoid region errors in the following example</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<'a> {} |
| impl<T> Trait<'static> for T {} |
| fn impls_trait<'a, T: Trait<'a>>() {} |
| fn foo<'a, T: Trait<'a>>() { |
| impls_trait::<'a, T>(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>We also need this as shadowed impls can result in currently ambiguous solver cycles: <a href="https://github.com/rust-lang/trait-system-refactor-initiative/issues/76">trait-system-refactor-initiative#76</a>. Without preference we'd be forced to fail with ambiguity |
| errors if the where-bound results in region constraints to avoid incompleteness.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Super { |
| type SuperAssoc; |
| } |
| |
| trait Trait: Super<SuperAssoc = Self::TraitAssoc> { |
| type TraitAssoc; |
| } |
| |
| impl<T, U> Trait for T |
| where |
| T: Super<SuperAssoc = U>, |
| { |
| type TraitAssoc = U; |
| } |
| |
| fn overflow<T: Trait>() { |
| // We can use the elaborated `Super<SuperAssoc = Self::TraitAssoc>` where-bound |
| // to prove the where-bound of the `T: Trait` implementation. This currently results in |
| // overflow. |
| let x: <T as Trait>::TraitAssoc; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This preference causes a lot of issues. See <a href="https://github.com/rust-lang/rust/issues/24066">#24066</a>. Most of the |
| issues are caused by preferring where-bounds over impls even if the where-bound guides type inference:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<T> { |
| fn call_me(&self, x: T) {} |
| } |
| impl<T> Trait<u32> for T {} |
| impl<T> Trait<i32> for T {} |
| fn bug<T: Trait<U>, U>(x: T) { |
| x.call_me(1u32); |
| //~^ ERROR mismatched types |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>However, even if we only apply this preference if the where-bound doesn't guide inference, it may still result |
| in incorrect lifetime constraints:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<'a> {} |
| impl<'a> Trait<'a> for &'a str {} |
| fn impls_trait<'a, T: Trait<'a>>(_: T) {} |
| fn foo<'a, 'b>(x: &'b str) |
| where |
| &'a str: Trait<'b> |
| { |
| // Need to prove `&'x str: Trait<'b>` with `'b: 'x`. |
| impls_trait::<'b, _>(x); |
| //~^ ERROR lifetime may not live long enough |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h4 id="preference-over-aliasbound-candidates"><a class="header" href="#preference-over-aliasbound-candidates">Preference over <code>AliasBound</code> candidates</a></h4> |
| <p>This is necessary to avoid region errors in the following example</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Bound<'a> {} |
| trait Trait<'a> { |
| type Assoc: Bound<'a>; |
| } |
| |
| fn impls_bound<'b, T: Bound<'b>>() {} |
| fn foo<'a, 'b, 'c, T>() |
| where |
| T: Trait<'a>, |
| for<'hr> T::Assoc: Bound<'hr>, |
| { |
| impls_bound::<'b, T::Assoc>(); |
| impls_bound::<'c, T::Assoc>(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>It can also result in unnecessary constraints</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Bound<'a> {} |
| trait Trait<'a> { |
| type Assoc: Bound<'a>; |
| } |
| |
| fn impls_bound<'b, T: Bound<'b>>() {} |
| fn foo<'a, 'b, T>() |
| where |
| T: for<'hr> Trait<'hr>, |
| <T as Trait<'b>>::Assoc: Bound<'a>, |
| { |
| // Using the where-bound for `<T as Trait<'a>>::Assoc: Bound<'a>` |
| // unnecessarily equates `<T as Trait<'a>>::Assoc` with the |
| // `<T as Trait<'b>>::Assoc` from the env. |
| impls_bound::<'a, <T as Trait<'a>>::Assoc>(); |
| // For a `<T as Trait<'b>>::Assoc: Bound<'b>` the self type of the |
| // where-bound matches, but the arguments of the trait bound don't. |
| impls_bound::<'b, <T as Trait<'b>>::Assoc>(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h4 id="why-no-preference-for-global-where-bounds"><a class="header" href="#why-no-preference-for-global-where-bounds">Why no preference for global where-bounds</a></h4> |
| <p>Global where-bounds are either fully implied by an impl or unsatisfiable. If they are unsatisfiable, we don't really care what happens. If a where-bound is fully implied then using the impl to prove the trait goal cannot result in additional constraints. For trait goals this is only useful for where-bounds which use <code>'static</code>:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait A { |
| fn test(&self); |
| } |
| |
| fn foo(x: &dyn A) |
| where |
| dyn A + 'static: A, // Using this bound would lead to a lifetime error. |
| { |
| x.test(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>More importantly, by using impls here we prevent global where-bounds from shadowing impls when normalizing associated types. There are no known issues from preferring impls over global where-bounds.</p> |
| <h4 id="why-still-consider-global-where-bounds"><a class="header" href="#why-still-consider-global-where-bounds">Why still consider global where-bounds</a></h4> |
| <p>Given that we just use impls even if there exists a global where-bounds, you may ask why we don't just ignore these global where-bounds entirely: we use them to weaken the inference guidance from non-global where-bounds.</p> |
| <p>Without a global where-bound, we currently prefer non-global where bounds even though there would be an applicable impl as well. By adding a non-global where-bound, this unnecessary inference guidance is disabled, allowing the following to compile:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn check<Color>(color: Color) |
| where |
| Vec: Into<Color> + Into<f32>, |
| { |
| let _: f32 = Vec.into(); |
| // Without the global `Vec: Into<f32>` bound we'd |
| // eagerly use the non-global `Vec: Into<Color>` bound |
| // here, causing this to fail. |
| } |
| |
| struct Vec; |
| impl From<Vec> for f32 { |
| fn from(_: Vec) -> Self { |
| loop {} |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="candidatesourcealiasbound"><a class="header" href="#candidatesourcealiasbound"><code>CandidateSource::AliasBound</code></a></h3> |
| <p>We prefer alias-bound candidates over impls. We currently use this preference to guide type inference, causing the following to compile. I personally don't think this preference is desirable 🤷</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>pub trait Dyn { |
| type Word: Into<u64>; |
| fn d_tag(&self) -> Self::Word; |
| fn tag32(&self) -> Option<u32> { |
| self.d_tag().into().try_into().ok() |
| // prove `Self::Word: Into<?0>` and then select a method |
| // on `?0`, needs eager inference. |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <pre><pre class="playground"><code class="language-rust">fn impl_trait() -> impl Into<u32> { |
| 0u16 |
| } |
| |
| fn main() { |
| // There are two possible types for `x`: |
| // - `u32` by using the "alias bound" of `impl Into<u32>` |
| // - `impl Into<u32>`, i.e. `u16`, by using `impl<T> From<T> for T` |
| // |
| // We infer the type of `x` to be `u32` even though this is not |
| // strictly necessary and can even lead to surprising errors. |
| let x = impl_trait().into(); |
| println!("{}", std::mem::size_of_val(&x)); |
| }</code></pre></pre> |
| <p>This preference also avoids ambiguity due to region constraints, I don't know whether people rely on this in practice.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Bound<'a> {} |
| impl<T> Bound<'static> for T {} |
| trait Trait<'a> { |
| type Assoc: Bound<'a>; |
| } |
| |
| fn impls_bound<'b, T: Bound<'b>>() {} |
| fn foo<'a, T: Trait<'a>>() { |
| // Should we infer this to `'a` or `'static`. |
| impls_bound::<'_, T::Assoc>(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="candidatesourcebuiltinimplbuiltinimplsourceobject_"><a class="header" href="#candidatesourcebuiltinimplbuiltinimplsourceobject_"><code>CandidateSource::BuiltinImpl(BuiltinImplSource::Object(_))</code></a></h3> |
| <p>We prefer builtin trait object impls over user-written impls. This is <strong>unsound</strong> and should be remoed in the future. See <a href="https://github.com/rust-lang/rust/issues/57893">#57893</a> and <a href="https://github.com/rust-lang/rust/pull/141347">#141347</a> for more details.</p> |
| <h2 id="normalizesto-goals"><a class="header" href="#normalizesto-goals"><code>NormalizesTo</code> goals</a></h2> |
| <p>The candidate preference behavior during normalization is implemented in <a href="https://github.com/rust-lang/rust/blob/e3ee7f7aea5b45af3b42b5e4713da43876a65ac9/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs#L920-L1003"><code>fn assemble_and_merge_candidates</code></a>.</p> |
| <h3 id="where-bounds-shadow-impls"><a class="header" href="#where-bounds-shadow-impls">Where-bounds shadow impls</a></h3> |
| <p>Normalization of associated items does not consider impls if the corresponding trait goal has been proven via a <code>ParamEnv</code> or <code>AliasBound</code> candidate. |
| This means that for where-bounds which do not constrain associated types, the associated types remain <em>rigid</em>.</p> |
| <p>This is necessary to avoid unnecessary region constraints from applying impls.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait<'a> { |
| type Assoc; |
| } |
| impl Trait<'static> for u32 { |
| type Assoc = u32; |
| } |
| |
| fn bar<'b, T: Trait<'b>>() -> T::Assoc { todo!() } |
| fn foo<'a>() |
| where |
| u32: Trait<'a>, |
| { |
| // Normalizing the return type would use the impl, proving |
| // the `T: Trait` where-bound would use the where-bound, resulting |
| // in different region constraints. |
| bar::<'_, u32>(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="we-always-consider-aliasbound-candidates"><a class="header" href="#we-always-consider-aliasbound-candidates">We always consider <code>AliasBound</code> candidates</a></h3> |
| <p>In case the where-bound does not specify the associated item, we consider <code>AliasBound</code> candidates instead of treating the alias as rigid, even though the trait goal was proven via a <code>ParamEnv</code> candidate.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Super { |
| type Assoc; |
| } |
| trait Bound { |
| type Assoc: Super<Assoc = u32>; |
| } |
| trait Trait: Super {} |
| |
| // Elaborating the environment results in a `T::Assoc: Super` where-bound. |
| // This where-bound must not prevent normalization via the `Super<Assoc = u32>` |
| // item bound. |
| fn heck<T: Bound<Assoc: Trait>>(x: <T::Assoc as Super>::Assoc) -> u32 { |
| x |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Using such an alias can result in additional region constraints, cc <a href="https://github.com/rust-lang/rust/issues/133044">#133044</a>.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Bound<'a> { |
| type Assoc; |
| } |
| trait Trait { |
| type Assoc: Bound<'static, Assoc = u32>; |
| } |
| |
| fn heck<'a, T: Trait<Assoc: Bound<'a>>>(x: <T::Assoc as Bound<'a>>::Assoc) { |
| // Normalizing the associated type requires `T::Assoc: Bound<'static>` as it |
| // uses the `Bound<'static>` alias-bound instead of keeping the alias rigid. |
| drop(x); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="we-prefer-paramenv-candidates-over-aliasbound"><a class="header" href="#we-prefer-paramenv-candidates-over-aliasbound">We prefer <code>ParamEnv</code> candidates over <code>AliasBound</code></a></h3> |
| <p>While we use <code>AliasBound</code> candidates if the where-bound does not specify the associated type, in case it does, we prefer the where-bound. |
| This is necessary for the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// Make sure we prefer the `I::IntoIterator: Iterator<Item = ()>` |
| // where-bound over the `I::Intoiterator: Iterator<Item = I::Item>` |
| // alias-bound. |
| |
| trait Iterator { |
| type Item; |
| } |
| |
| trait IntoIterator { |
| type Item; |
| type IntoIter: Iterator<Item = Self::Item>; |
| } |
| |
| fn normalize<I: Iterator<Item = ()>>() {} |
| |
| fn foo<I>() |
| where |
| I: IntoIterator, |
| I::IntoIter: Iterator<Item = ()>, |
| { |
| // We need to prefer the `I::IntoIterator: Iterator<Item = ()>` |
| // where-bound over the `I::Intoiterator: Iterator<Item = I::Item>` |
| // alias-bound. |
| normalize::<I::IntoIter>(); |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h3 id="we-always-consider-where-bounds"><a class="header" href="#we-always-consider-where-bounds">We always consider where-bounds</a></h3> |
| <p>Even if the trait goal was proven via an impl, we still prefer <code>ParamEnv</code> candidates, if any exist.</p> |
| <h4 id="we-prefer-orphaned-where-bounds"><a class="header" href="#we-prefer-orphaned-where-bounds">We prefer "orphaned" where-bounds</a></h4> |
| <p>We add "orphaned" <code>Projection</code> clauses into the <code>ParamEnv</code> when normalizing item bounds of GATs and RPITIT in <code>fn check_type_bounds</code>. |
| We need to prefer these <code>ParamEnv</code> candidates over impls and other where-bounds.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(associated_type_defaults)] |
| <span class="boring">fn main() { |
| </span>trait Foo { |
| // We should be able to prove that `i32: Baz<Self>` because of |
| // the impl below, which requires that `Self::Bar<()>: Eq<i32>` |
| // which is true, because we assume `for<T> Self::Bar<T> = i32`. |
| type Bar<T>: Baz<Self> = i32; |
| } |
| trait Baz<T: ?Sized> {} |
| impl<T: Foo + ?Sized> Baz<T> for i32 where T::Bar<()>: Eq<i32> {} |
| trait Eq<T> {} |
| impl<T> Eq<T> for T {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>I don't fully understand the cases where this preference is actually necessary and haven't been able to exploit this in fun ways yet, but 🤷</p> |
| <h4 id="we-prefer-global-where-bounds-over-impls"><a class="header" href="#we-prefer-global-where-bounds-over-impls">We prefer global where-bounds over impls</a></h4> |
| <p>This is necessary for the following to compile. I don't know whether anything relies on it in practice 🤷</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Id { |
| type This; |
| } |
| impl<T> Id for T { |
| type This = T; |
| } |
| |
| fn foo<T>(x: T) -> <u32 as Id>::This |
| where |
| u32: Id<This = T>, |
| { |
| x |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This means normalization can result in additional region constraints, cc <a href="https://github.com/rust-lang/rust/issues/133044">#133044</a>.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait { |
| type Assoc; |
| } |
| |
| impl Trait for &u32 { |
| type Assoc = u32; |
| } |
| |
| fn trait_bound<T: Trait>() {} |
| fn normalize<T: Trait<Assoc = u32>>() {} |
| |
| fn foo<'a>() |
| where |
| &'static u32: Trait<Assoc = u32>, |
| { |
| trait_bound::<&'a u32>(); // ok, proven via impl |
| normalize::<&'a u32>(); // error, proven via where-bound |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="canonicalization-1"><a class="header" href="#canonicalization-1">Canonicalization</a></h1> |
| <p>Canonicalization is the process of <em>isolating</em> a value from its context and is necessary |
| for global caching of goals which include inference variables.</p> |
| <p>The idea is that given the goals <code>u32: Trait<?x></code> and <code>u32: Trait<?y></code>, where <code>?x</code> and <code>?y</code> |
| are two different currently unconstrained inference variables, we should get the same result |
| for both goals. We can therefore prove <em>the canonical query</em> <code>exists<T> u32: Trait<T></code> once |
| and reuse the result.</p> |
| <p>Let's first go over the way canonical queries work and then dive into the specifics of |
| how canonicalization works.</p> |
| <h2 id="a-walkthrough-of-canonical-queries"><a class="header" href="#a-walkthrough-of-canonical-queries">A walkthrough of canonical queries</a></h2> |
| <p>To make this a bit easier, let's use the trait goal <code>u32: Trait<?x></code> as an example with the |
| assumption that the only relevant impl is <code>impl<T> Trait<Vec<T>> for u32</code>.</p> |
| <h3 id="canonicalizing-the-input"><a class="header" href="#canonicalizing-the-input">Canonicalizing the input</a></h3> |
| <p>We start by <em>canonicalizing</em> the goal, replacing inference variables with existential and |
| placeholders with universal bound variables. This would result in the <em>canonical goal</em> |
| <code>exists<T> u32: Trait<T></code>.</p> |
| <p>We remember the original values of all bound variables in the original context. Here this would |
| map <code>T</code> back to <code>?x</code>. These original values are used later on when dealing with the query |
| response.</p> |
| <p>We now call the canonical query with the canonical goal.</p> |
| <h3 id="instantiating-the-canonical-goal-inside-of-the-query"><a class="header" href="#instantiating-the-canonical-goal-inside-of-the-query">Instantiating the canonical goal inside of the query</a></h3> |
| <p>To actually try to prove the canonical goal we start by instantiating the bound variables with |
| inference variables and placeholders again.</p> |
| <p>This happens inside of the query in a completely separate <code>InferCtxt</code>. Inside of the query we |
| now have a goal <code>u32: Trait<?0></code>. We also remember which value we've used to instantiate the bound |
| variables in the canonical goal, which maps <code>T</code> to <code>?0</code>.</p> |
| <p>We now compute the goal <code>u32: Trait<?0></code> and figure out that this holds, but we've constrained |
| <code>?0</code> to <code>Vec<?1></code>. We finally convert this result to something useful to the caller.</p> |
| <h3 id="canonicalizing-the-query-response"><a class="header" href="#canonicalizing-the-query-response">Canonicalizing the query response</a></h3> |
| <p>We have to return to the caller both whether the goal holds, and the inference constraints |
| from inside of the query.</p> |
| <p>To return the inference results to the caller we canonicalize the mapping from bound variables |
| to the instantiated values in the query. This means that the query response is <code>Certainty::Yes</code> |
| and a mapping from <code>T</code> to <code>exists<U> Vec<U></code>.</p> |
| <h3 id="instantiating-the-query-response"><a class="header" href="#instantiating-the-query-response">Instantiating the query response</a></h3> |
| <p>The caller now has to apply the constraints returned by the query. For this they first |
| instantiate the bound variables of the canonical response with inference variables and |
| placeholders again, so the mapping in the response is now from <code>T</code> to <code>Vec<?z></code>.</p> |
| <p>It now equates the original value of <code>T</code> (<code>?x</code>) with the value for <code>T</code> in the |
| response (<code>Vec<?z></code>), which correctly constrains <code>?x</code> to <code>Vec<?z></code>.</p> |
| <h2 id="externalconstraints"><a class="header" href="#externalconstraints"><code>ExternalConstraints</code></a></h2> |
| <p>Computing a trait goal may not only constrain inference variables, it can also add region |
| obligations, e.g. given a goal <code>(): AOutlivesB<'a, 'b></code> we would like to return the fact that |
| <code>'a: 'b</code> has to hold.</p> |
| <p>This is done by not only returning the mapping from bound variables to the instantiated values |
| from the query but also extracting additional <code>ExternalConstraints</code> from the <code>InferCtxt</code> context |
| while building the response.</p> |
| <h2 id="how-exactly-does-canonicalization-work"><a class="header" href="#how-exactly-does-canonicalization-work">How exactly does canonicalization work</a></h2> |
| <p>TODO: link to code once the PR lands and elaborate</p> |
| <ul> |
| <li>types and consts: infer to existentially bound var, placeholder to universally bound var, |
| considering universes</li> |
| <li>generic parameters in the input get treated as placeholders in the root universe</li> |
| <li>all regions in the input get all mapped to existentially bound vars and we "uniquify" them. |
| <code>&'a (): Trait<'a></code> gets canonicalized to <code>exists<'0, '1> &'0 (): Trait<'1></code>. We do not care |
| about their universes and simply put all regions into the highest universe of the input.</li> |
| <li>in the output everything in a universe of the caller gets put into the root universe and only |
| gets its correct universe when we unify the var values with the orig values of the caller</li> |
| <li>we do not uniquify regions in the response and don't canonicalize <code>'static</code></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="coinduction"><a class="header" href="#coinduction">Coinduction</a></h1> |
| <p>The trait solver may use coinduction when proving goals. |
| Coinduction is fairly subtle so we're giving it its own chapter.</p> |
| <h2 id="coinduction-and-induction"><a class="header" href="#coinduction-and-induction">Coinduction and induction</a></h2> |
| <p>With induction, we recursively apply proofs until we end up with a finite proof tree. |
| Consider the example of <code>Vec<Vec<Vec<u32>>>: Debug</code> which results in the following tree.</p> |
| <ul> |
| <li><code>Vec<Vec<Vec<u32>>>: Debug</code> |
| <ul> |
| <li><code>Vec<Vec<u32>>: Debug</code> |
| <ul> |
| <li><code>Vec<u32>: Debug</code> |
| <ul> |
| <li><code>u32: Debug</code></li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p>This tree is finite. But not all goals we would want to hold have finite proof trees, |
| consider the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct List<T> { |
| value: T, |
| next: Option<Box<List<T>>>, |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>For <code>List<T>: Send</code> to hold all its fields have to recursively implement <code>Send</code> as well. |
| This would result in the following proof tree:</p> |
| <ul> |
| <li><code>List<T>: Send</code> |
| <ul> |
| <li><code>T: Send</code></li> |
| <li><code>Option<Box<List<T>>>: Send</code> |
| <ul> |
| <li><code>Box<List<T>>: Send</code> |
| <ul> |
| <li><code>List<T>: Send</code> |
| <ul> |
| <li><code>T: Send</code></li> |
| <li><code>Option<Box<List<T>>>: Send</code> |
| <ul> |
| <li><code>Box<List<T>>: Send</code> |
| <ul> |
| <li>...</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p>This tree would be infinitely large which is exactly what coinduction is about.</p> |
| <blockquote> |
| <p>To <strong>inductively</strong> prove a goal you need to provide a finite proof tree for it. |
| To <strong>coinductively</strong> prove a goal the provided proof tree may be infinite.</p> |
| </blockquote> |
| <h2 id="why-is-coinduction-correct"><a class="header" href="#why-is-coinduction-correct">Why is coinduction correct</a></h2> |
| <p>When checking whether some trait goals holds, we're asking "does there exist an <code>impl</code> |
| which satisfies this bound". Even if are infinite chains of nested goals, we still have a |
| unique <code>impl</code> which should be used.</p> |
| <h2 id="how-to-implement-coinduction"><a class="header" href="#how-to-implement-coinduction">How to implement coinduction</a></h2> |
| <p>While our implementation can not check for coinduction by trying to construct an infinite |
| tree as that would take infinite resources, it still makes sense to think of coinduction |
| from this perspective.</p> |
| <p>As we cannot check for infinite trees, we instead search for patterns for which we know that |
| they would result in an infinite proof tree. The currently pattern we detect are (canonical) |
| cycles. If <code>T: Send</code> relies on <code>T: Send</code> then it's pretty clear that this will just go on forever.</p> |
| <p>With cycles we have to be careful with caching. Because of canonicalization of regions and |
| inference variables encountering a cycle doesn't mean that we would get an infinite proof tree. |
| Looking at the following example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo {} |
| struct Wrapper<T>(T); |
| |
| impl<T> Foo for Wrapper<Wrapper<T>> |
| where |
| Wrapper<T>: Foo |
| {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>Proving <code>Wrapper<?0>: Foo</code> uses the impl <code>impl<T> Foo for Wrapper<Wrapper<T>></code> which constrains |
| <code>?0</code> to <code>Wrapper<?1></code> and then requires <code>Wrapper<?1>: Foo</code>. Due to canonicalization this would be |
| detected as a cycle.</p> |
| <p>The idea to solve is to return a <em>provisional result</em> whenever we detect a cycle and repeatedly |
| retry goals until the <em>provisional result</em> is equal to the final result of that goal. We |
| start out by using <code>Yes</code> with no constraints as the result and then update it to the result of |
| the previous iteration whenever we have to rerun.</p> |
| <p>TODO: elaborate here. We use the same approach as chalk for coinductive cycles. |
| Note that the treatment for inductive cycles currently differs by simply returning <code>Overflow</code>. |
| See <a href="https://rust-lang.github.io/chalk/book/recursive/inductive_cycles.html">the relevant chapters</a> in the chalk book.</p> |
| <h2 id="future-work"><a class="header" href="#future-work">Future work</a></h2> |
| <p>We currently only consider auto-traits, <code>Sized</code>, and <code>WF</code>-goals to be coinductive. |
| In the future we pretty much intend for all goals to be coinductive. |
| Lets first elaborate on why allowing more coinductive proofs is even desirable.</p> |
| <h3 id="recursive-data-types-already-rely-on-coinduction"><a class="header" href="#recursive-data-types-already-rely-on-coinduction">Recursive data types already rely on coinduction...</a></h3> |
| <p>...they just tend to avoid them in the trait solver.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>enum List<T> { |
| Nil, |
| Succ(T, Box<List<T>>), |
| } |
| |
| impl<T: Clone> Clone for List<T> { |
| fn clone(&self) -> Self { |
| match self { |
| List::Nil => List::Nil, |
| List::Succ(head, tail) => List::Succ(head.clone(), tail.clone()), |
| } |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>We are using <code>tail.clone()</code> in this impl. For this we have to prove <code>Box<List<T>>: Clone</code> |
| which requires <code>List<T>: Clone</code> but that relies on the impl which we are currently checking. |
| By adding that requirement to the <code>where</code>-clauses of the impl, which is what we would |
| do with <a href="https://smallcultfollowing.com/babysteps/blog/2022/04/12/implied-bounds-and-perfect-derive">perfect derive</a>, we move that cycle into the trait solver and <a href="https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=0a9c3830b93a2380e6978d6328df8f72">get an error</a>.</p> |
| <h3 id="recursive-data-types"><a class="header" href="#recursive-data-types">Recursive data types</a></h3> |
| <p>We also need coinduction to reason about recursive types containing projections, |
| e.g. the following currently fails to compile even though it should be valid.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>use std::borrow::Cow; |
| pub struct Foo<'a>(Cow<'a, [Foo<'a>]>); |
| <span class="boring">}</span></code></pre></pre> |
| <p>This issue has been known since at least 2015, see |
| <a href="https://github.com/rust-lang/rust/issues/23714">#23714</a> if you want to know more.</p> |
| <h3 id="explicitly-checked-implied-bounds"><a class="header" href="#explicitly-checked-implied-bounds">Explicitly checked implied bounds</a></h3> |
| <p>When checking an impl, we assume that the types in the impl headers are well-formed. |
| This means that when using instantiating the impl we have to prove that's actually the case. |
| <a href="https://github.com/rust-lang/rust/issues/100051">#100051</a> shows that this is not the case. |
| To fix this, we have to add <code>WF</code> predicates for the types in impl headers. |
| Without coinduction for all traits, this even breaks <code>core</code>.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait FromResidual<R> {} |
| trait Try: FromResidual<<Self as Try>::Residual> { |
| type Residual; |
| } |
| |
| struct Ready<T>(T); |
| impl<T> Try for Ready<T> { |
| type Residual = Ready<()>; |
| } |
| impl<T> FromResidual<<Ready<T> as Try>::Residual> for Ready<T> {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>When checking that the impl of <code>FromResidual</code> is well formed we get the following cycle:</p> |
| <p>The impl is well formed if <code><Ready<T> as Try>::Residual</code> and <code>Ready<T></code> are well formed.</p> |
| <ul> |
| <li><code>wf(<Ready<T> as Try>::Residual)</code> requires</li> |
| <li><code>Ready<T>: Try</code>, which requires because of the super trait</li> |
| <li><code>Ready<T>: FromResidual<Ready<T> as Try>::Residual></code>, <strong>because of implied bounds on impl</strong></li> |
| <li><code>wf(<Ready<T> as Try>::Residual)</code> :tada: <strong>cycle</strong></li> |
| </ul> |
| <h3 id="issues-when-extending-coinduction-to-more-goals"><a class="header" href="#issues-when-extending-coinduction-to-more-goals">Issues when extending coinduction to more goals</a></h3> |
| <p>There are some additional issues to keep in mind when extending coinduction. |
| The issues here are not relevant for the current solver.</p> |
| <h4 id="implied-super-trait-bounds"><a class="header" href="#implied-super-trait-bounds">Implied super trait bounds</a></h4> |
| <p>Our trait system currently treats super traits, e.g. <code>trait Trait: SuperTrait</code>, |
| by 1) requiring that <code>SuperTrait</code> has to hold for all types which implement <code>Trait</code>, |
| and 2) assuming <code>SuperTrait</code> holds if <code>Trait</code> holds.</p> |
| <p>Relying on 2) while proving 1) is unsound. This can only be observed in case of |
| coinductive cycles. Without cycles, whenever we rely on 2) we must have also |
| proven 1) without relying on 2) for the used impl of <code>Trait</code>.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait: SuperTrait {} |
| |
| impl<T: Trait> Trait for T {} |
| |
| // Keeping the current setup for coinduction |
| // would allow this compile. Uff :< |
| fn sup<T: SuperTrait>() {} |
| fn requires_trait<T: Trait>() { sup::<T>() } |
| fn generic<T>() { requires_trait::<T>() } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This is not really fundamental to coinduction but rather an existing property |
| which is made unsound because of it.</p> |
| <h5 id="possible-solutions"><a class="header" href="#possible-solutions">Possible solutions</a></h5> |
| <p>The easiest way to solve this would be to completely remove 2) and always elaborate |
| <code>T: Trait</code> to <code>T: Trait</code> and <code>T: SuperTrait</code> outside of the trait solver. |
| This would allow us to also remove 1), but as we still have to prove ordinary |
| <code>where</code>-bounds on traits, that's just additional work.</p> |
| <p>While one could imagine ways to disable cyclic uses of 2) when checking 1), |
| at least the ideas of myself - @lcnr - are all far to complex to be reasonable.</p> |
| <h4 id="normalizes_to-goals-and-progress"><a class="header" href="#normalizes_to-goals-and-progress"><code>normalizes_to</code> goals and progress</a></h4> |
| <p>A <code>normalizes_to</code> goal represents the requirement that <code><T as Trait>::Assoc</code> normalizes |
| to some <code>U</code>. This is achieved by defacto first normalizing <code><T as Trait>::Assoc</code> and then |
| equating the resulting type with <code>U</code>. It should be a mapping as each projection should normalize |
| to exactly one type. By simply allowing infinite proof trees, we would get the following behavior:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Trait { |
| type Assoc; |
| } |
| |
| impl Trait for () { |
| type Assoc = <() as Trait>::Assoc; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>If we now compute <code>normalizes_to(<() as Trait>::Assoc, Vec<u32>)</code>, we would resolve the impl |
| and get the associated type <code><() as Trait>::Assoc</code>. We then equate that with the expected type, |
| causing us to check <code>normalizes_to(<() as Trait>::Assoc, Vec<u32>)</code> again. |
| This just goes on forever, resulting in an infinite proof tree.</p> |
| <p>This means that <code><() as Trait>::Assoc</code> would be equal to any other type which is unsound.</p> |
| <h5 id="how-to-solve-this"><a class="header" href="#how-to-solve-this">How to solve this</a></h5> |
| <p><strong>WARNING: THIS IS SUBTLE AND MIGHT BE WRONG</strong></p> |
| <p>Unlike trait goals, <code>normalizes_to</code> has to be <em>productive</em><sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup>. A <code>normalizes_to</code> goal |
| is productive once the projection normalizes to a rigid type constructor, |
| so <code><() as Trait>::Assoc</code> normalizing to <code>Vec<<() as Trait>::Assoc></code> would be productive.</p> |
| <p>A <code>normalizes_to</code> goal has two kinds of nested goals. Nested requirements needed to actually |
| normalize the projection, and the equality between the normalized projection and the |
| expected type. Only the equality has to be productive. A branch in the proof tree is productive |
| if it is either finite, or contains at least one <code>normalizes_to</code> where the alias is resolved |
| to a rigid type constructor.</p> |
| <p>Alternatively, we could simply always treat the equate branch of <code>normalizes_to</code> as inductive. |
| Any cycles should result in infinite types, which aren't supported anyways and would only |
| result in overflow when deeply normalizing for codegen.</p> |
| <p>experimentation and examples: <a href="https://hackmd.io/-8p0AHnzSq2VAE6HE_wX-w?view">https://hackmd.io/-8p0AHnzSq2VAE6HE_wX-w?view</a></p> |
| <p>Another attempt at a summary.</p> |
| <ul> |
| <li>in projection eq, we must make progress with constraining the rhs</li> |
| <li>a cycle is only ok if while equating we have a rigid ty on the lhs after norm at least once</li> |
| <li>cycles outside of the recursive <code>eq</code> call of <code>normalizes_to</code> are always fine</li> |
| </ul> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>related: <a href="https://coq.inria.fr/refman/language/core/coinductive.html#top-level-definitions-of-corecursive-functions">https://coq.inria.fr/refman/language/core/coinductive.html#top-level-definitions-of-corecursive-functions</a> <a href="#fr-1-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="caching-in-the-new-trait-solver"><a class="header" href="#caching-in-the-new-trait-solver">Caching in the new trait solver</a></h1> |
| <p>Caching results of the trait solver is necessary for performance. |
| We have to make sure that it is sound. Caching is handled by the |
| <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L102-L117"><code>SearchGraph</code></a></p> |
| <h2 id="the-global-cache"><a class="header" href="#the-global-cache">The global cache</a></h2> |
| <p>At its core, the cache is fairly straightforward. When evaluating a goal, we |
| check whether it's in the global cache. If so, we reuse that entry. If not, we |
| compute the goal and then store its result in the cache.</p> |
| <p>To handle incremental compilation the computation of a goal happens inside of |
| <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L391"><code>DepGraph::with_anon_task</code></a> which creates a new <code>DepNode</code> which depends on all queries |
| used inside of this computation. When accessing the global cache we then read this |
| <code>DepNode</code>, manually adding a dependency edge to all the queries used: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L78">source</a>.</p> |
| <h3 id="dealing-with-overflow"><a class="header" href="#dealing-with-overflow">Dealing with overflow</a></h3> |
| <p>Hitting the recursion limit is not fatal in the new trait solver but instead simply |
| causes it to return ambiguity: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L276">source</a>. Whether we hit the recursion limit |
| can therefore change the result without resulting in a compilation failure. This |
| means we must consider the remaining available depth when accessing a cache result.</p> |
| <p>We do this by storing more information in the cache entry. For goals whose evaluation |
| did not reach the recursion limit, we simply store its reached depth: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L102">source</a>. |
| These results can freely be used as long as the current <code>available_depth</code> is higher than |
| its <code>reached_depth</code>: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L76-L86">source</a>. We then update the reached depth of the |
| current goal to make sure that whether we've used the global cache entry is not |
| observable: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L308">source</a>.</p> |
| <p>For goals which reach the recursion limit we currently only use the cached result if the |
| available depth <em>exactly matches</em> the depth of the entry. The cache entry for each goal |
| therefore contains a separate result for each remaining depth: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L124">source</a>.<sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup></p> |
| <h2 id="handling-cycles"><a class="header" href="#handling-cycles">Handling cycles</a></h2> |
| <p>The trait solver has to support cycles. These cycles are either inductive or coinductive, |
| depending on the participating goals. See the <a href="solve/./coinduction.html">chapter on coinduction</a> for more details. |
| We distinguish between the cycle heads and the cycle root: a stack entry is a |
| cycle head if it recursively accessed. The <em>root</em> is the deepest goal on the stack which |
| is involved in any cycle. Given the following dependency tree, <code>A</code> and <code>B</code> are both cycle |
| heads, while only <code>A</code> is a root.</p> |
| <pre class="mermaid">graph TB |
| A --> B |
| B --> C |
| C --> B |
| C --> A |
| </pre> |
| <p>The result of cycle participants depends on the result of goals still on the stack. |
| However, we are currently computing that result, so its result is still unknown. This is |
| handled by evaluating cycle heads until we reach a fixpoint. In the first iteration, we |
| return either success or overflow with no constraints, depending on whether the cycle is |
| coinductive: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L366-L370">source</a>. After evaluating the head of a cycle, we |
| check whether its <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L57"><code>provisional_result</code></a> is equal to the result of this iteration. If so, |
| we've finished evaluating this cycle and return its result. If not, we update the provisional |
| result and reevaluate the goal: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L425-L446">source</a>. After the first iteration it does not |
| matter whether cycles are coinductive or inductive. We always use the provisional result.</p> |
| <h3 id="only-caching-cycle-roots"><a class="header" href="#only-caching-cycle-roots">Only caching cycle roots</a></h3> |
| <p>We cannot move the result of any cycle participant to the global cache until we've |
| finished evaluating the cycle root. However, even after we've completely evaluated the |
| cycle, we are still forced to discard the result of all participants apart from the root |
| itself.</p> |
| <p>We track the query dependencies of all global cache entries. This causes the caching of |
| cycle participants to be non-trivial. We cannot simply reuse the <code>DepNode</code> of the cycle |
| root.<sup class="footnote-reference" id="fr-2-1"><a href="#footnote-2">2</a></sup> If we have a cycle <code>A -> B -> A</code>, then the <code>DepNode</code> for <code>A</code> contains a dependency |
| from <code>A -> B</code>. Reusing this entry for <code>B</code> may break if the source is changed. The <code>B -> A</code> |
| edge may not exist anymore and <code>A</code> may have been completely removed. This can easily result |
| in an ICE.</p> |
| <p>However, it's even worse as the result of a cycle can change depending on which goal is |
| the root: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/tests/ui/traits/next-solver/cycles/coinduction/incompleteness-unstable-result.rs#L4-L16">example</a>. This forces us to weaken caching even further. |
| We must not use a cache entry of a cycle root, if there exists a stack entry, which was |
| a participant of its cycle involving that root. We do this by storing all cycle participants |
| of a given root in its global cache entry and checking that it contains no element of the |
| stack: <a href="https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L72-L74">source</a>.</p> |
| <h3 id="the-provisional-cache"><a class="header" href="#the-provisional-cache">The provisional cache</a></h3> |
| <p>TODO: write this :3</p> |
| <ul> |
| <li>stack dependence of provisional results</li> |
| <li>edge case: provisional cache impacts behavior</li> |
| </ul> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>This is overly restrictive: if all nested goals return the overflow response with some |
| available depth <code>n</code>, then their result should be the same for any depths smaller than <code>n</code>. |
| We can implement this optimization in the future. <a href="#fr-1-1">↩</a></p> |
| </li> |
| <li id="footnote-2"> |
| <p>summarizing the relevant <a href="https://rust-lang.zulipchat.com/#narrow/stream/364551-t-types.2Ftrait-system-refactor/topic/global.20cache">Zulip thread</a> <a href="#fr-2-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="proof-trees"><a class="header" href="#proof-trees">Proof trees</a></h1> |
| <p>While the trait solver itself only returns whether a goal holds and the necessary |
| constraints, we sometimes also want to know what happened while trying to prove |
| it. While the trait solver should generally be treated as a black box by the rest |
| of the compiler, we cannot completely ignore its internals and provide "proof trees" |
| as an interface for this. To use them you implement the <a href="https://github.com/rust-lang/rust/blob/d6c8169c186ab16a3404cd0d0866674018e8a19e/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs#L403"><code>ProofTreeVisitor</code></a> trait, |
| see its existing implementations for examples. The most notable uses are to compute |
| the <a href="https://github.com/rust-lang/rust/blob/d6c8169c186ab16a3404cd0d0866674018e8a19e/compiler/rustc_trait_selection/src/traits/coherence.rs#L742-L748">intercrate ambiguity causes for coherence errors</a>, |
| <a href="https://github.com/rust-lang/rust/blob/d6c8169c186ab16a3404cd0d0866674018e8a19e/compiler/rustc_trait_selection/src/solve/fulfill.rs#L343-L356">improving trait solver errors</a>, and |
| <a href="https://github.com/rust-lang/rust/blob/d6c8169c186ab16a3404cd0d0866674018e8a19e/compiler/rustc_hir_typeck/src/closure.rs#L333-L339">eagerly inferring closure signatures</a>.</p> |
| <h2 id="computing-proof-trees"><a class="header" href="#computing-proof-trees">Computing proof trees</a></h2> |
| <p>The trait solver uses <a href="solve/./canonicalization.html">Canonicalization</a> and uses completely separate <code>InferCtxt</code> for |
| each nested goal. Both diagnostics and auto-traits in rustdoc need to correctly |
| handle "looking into nested goals". Given a goal like <code>Vec<Vec<?x>>: Debug</code>, we |
| canonicalize to <code>exists<T0> Vec<Vec<T0>>: Debug</code>, instantiate that goal as |
| <code>Vec<Vec<?0>>: Debug</code>, get a nested goal <code>Vec<?0>: Debug</code>, canonicalize this to get |
| <code>exists<T0> Vec<T0>: Debug</code>, instantiate this as <code>Vec<?0>: Debug</code> which then results |
| in a nested <code>?0: Debug</code> goal which is ambiguous.</p> |
| <p>We compute proof trees by passing a <a href="https://github.com/rust-lang/rust/blob/d6c8169c186ab16a3404cd0d0866674018e8a19e/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs#L40"><code>ProofTreeBuilder</code></a> to the search graph which is |
| converting the evaluation steps of the trait solver into a tree. When storing any |
| data using inference variables or placeholders, the data is canonicalized together |
| with the list of all unconstrained inference variables created during this computation. |
| This <a href="https://github.com/rust-lang/rust/blob/d6c8169c186ab16a3404cd0d0866674018e8a19e/compiler/rustc_type_ir/src/solve/inspect.rs#L31-L47"><code>CanonicalState</code></a> is then instantiated in the parent inference context while |
| walking the proof tree, using the list of inference variables to connect all the |
| canonicalized values created during this evaluation.</p> |
| <h2 id="debugging-the-solver"><a class="header" href="#debugging-the-solver">Debugging the solver</a></h2> |
| <p>We previously also tried to use proof trees to debug the solver implementation. This |
| has different design requirements than analyzing it programmatically. The recommended |
| way to debug the trait solver is by using <code>tracing</code>. The trait solver only uses the |
| <code>debug</code> tracing level for its general 'shape' and <code>trace</code> for additional detail. |
| <code>RUSTC_LOG=rustc_next_trait_solver=debug</code> therefore gives you a general outline |
| and <code>RUSTC_LOG=rustc_next_trait_solver=trace</code> can then be used if more precise |
| information is required.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="opaque-types-in-the-new-solver"><a class="header" href="#opaque-types-in-the-new-solver">Opaque types in the new solver</a></h1> |
| <p>The way <a href="solve/../opaque-types-type-alias-impl-trait.html">opaque types</a> are handled in the new solver differs from the old implementation. |
| This should be a self-contained explanation of the behavior in the new solver.</p> |
| <h2 id="opaques-are-alias-types"><a class="header" href="#opaques-are-alias-types">opaques are alias types</a></h2> |
| <p>Opaque types are treated the same as other aliases, most notabily associated types, |
| whenever possible. There should be as few divergences in behavior as possible.</p> |
| <p>This is desirable, as they are very similar to other alias types, in that they can be |
| normalized to their hidden type and also have the same requirements for completeness. |
| Treating them this way also reduces the complexity of the type system by sharing code. |
| Having to deal with opaque types separately results in more complex rules and new kinds |
| of interactions. As we need to treat them like other aliases in the implicit-negative |
| mode, having significant differences between modes also adds complexity.</p> |
| <p><em>open question: is there an alternative approach here, maybe by treating them more like rigid |
| types with more limited places to instantiate them? they would still have to be ordinary |
| aliases during coherence</em></p> |
| <h3 id="normalizes-to-for-opaques"><a class="header" href="#normalizes-to-for-opaques"><code>normalizes-to</code> for opaques</a></h3> |
| <p><a href="https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L13">source</a></p> |
| <p><code>normalizes-to</code> is used to define the one-step normalization behavior for aliases in the new |
| solver: <code><<T as IdInner>::Assoc as IdOuter>::Assoc</code> first normalizes to <code><T as IdInner>::Assoc</code> |
| which then normalizes to <code>T</code>. It takes both the <code>AliasTy</code> which is getting normalized and the |
| expected <code>Term</code>. To use <code>normalizes-to</code> for actual normalization, the expected term can simply |
| be an unconstrained inference variable.</p> |
| <p>For opaque types in the defining scope and in the implicit-negative coherence mode, this is |
| always done in two steps. Outside of the defining scope <code>normalizes-to</code> for opaques always |
| returns <code>Err(NoSolution)</code>.</p> |
| <p>We start by trying to assign the expected type as a hidden type.</p> |
| <p>In the implicit-negative coherence mode, this currently always results in ambiguity without |
| interacting with the opaque types storage. We could instead add allow 'defining' all opaque types, |
| discarding their inferred types at the end, changing the behavior of an opaque type is used |
| multiple times during coherence: <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui/type-alias-impl-trait/coherence/coherence_different_hidden_ty.rs">example</a></p> |
| <p>Inside of the defining scope we start by checking whether the type and const arguments of the |
| opaque are all placeholders: <a href="https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L33">source</a>. If this check is ambiguous, |
| return ambiguity, if it fails, return <code>Err(NoSolution)</code>. This check ignores regions which are |
| only checked at the end of borrowck. If it succeeds, continue.</p> |
| <p>We then check whether we're able to <em>semantically</em> unify the generic arguments of the opaque |
| with the arguments of any opaque type already in the opaque types storage. If so, we unify the |
| previously stored type with the expected type of this <code>normalizes-to</code> call: <a href="https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L51-L59">source</a><sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup>.</p> |
| <p>If not, we insert the expected type in the opaque types storage: <a href="https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L68">source</a><sup class="footnote-reference" id="fr-2-1"><a href="#footnote-2">2</a></sup>. |
| Finally, we check whether the item bounds of the opaque hold for the expected type: |
| <a href="https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L69-L74">source</a>.</p> |
| <h3 id="using-alias-bounds-of-normalizable-aliases"><a class="header" href="#using-alias-bounds-of-normalizable-aliases">using alias-bounds of normalizable aliases</a></h3> |
| <p>https://github.com/rust-lang/trait-system-refactor-initiative/issues/77</p> |
| <p>Using an <code>AliasBound</code> candidate for normalizable aliases is generally not possible as an |
| associated type can have stronger bounds then the resulting type when normalizing via a |
| <code>ParamEnv</code> candidate.</p> |
| <p>These candidates would change our exact normalization strategy to be user-facing. It is otherwise |
| pretty much unobservable whether we eagerly normalize. Where we normalize is something we likely |
| want to change that after removing support for the old solver, so that would be undesirable.</p> |
| <h2 id="opaque-types-can-be-defined-anywhere"><a class="header" href="#opaque-types-can-be-defined-anywhere">opaque types can be defined anywhere</a></h2> |
| <p>Opaque types in their defining-scope can be defined anywhere, whether when simply relating types |
| or in the trait solver. This removes order dependence and incompleteness. Without this the result |
| of a goal can differ due to subtle reasons, e.g. whether we try to evaluate a goal using the |
| opaque before the first defining use of the opaque.</p> |
| <h2 id="higher-ranked-opaque-types-in-their-defining-scope"><a class="header" href="#higher-ranked-opaque-types-in-their-defining-scope">higher ranked opaque types in their defining scope</a></h2> |
| <p>These are not supported and trying to define them right now should always error.</p> |
| <p>FIXME: Because looking up opaque types in the opaque type storage can now unify regions, |
| we have to eagerly check that the opaque types does not reference placeholders. We otherwise |
| end up leaking placeholders.</p> |
| <h2 id="member-constraints"><a class="header" href="#member-constraints">member constraints</a></h2> |
| <p>The handling of member constraints does not change in the new solver. See the |
| <a href="solve/../borrow_check/region_inference/member_constraints.html">relevant existing chapter</a> for that.</p> |
| <h2 id="calling-methods-on-opaque-types"><a class="header" href="#calling-methods-on-opaque-types">calling methods on opaque types</a></h2> |
| <p>FIXME: We need to continue to support calling methods on still unconstrained |
| opaque types in their defining scope. It's unclear how to best do this.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>use std::future::Future; |
| use futures::FutureExt; |
| |
| fn go(i: usize) -> impl Future<Output = ()> + Send + 'static { |
| async move { |
| if i != 0 { |
| // This returns `impl Future<Output = ()>` in its defining scope, |
| // we don't know the concrete type of that opaque at this point. |
| // Currently treats the opaque as a known type and succeeds, but |
| // from the perspective of "easiest to soundly implement", it would |
| // be good for this to be ambiguous. |
| go(i - 1).boxed().await; |
| } |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>FIXME: this should ideally only result in a unique candidate given that we require the args to be placeholders and regions are always inference vars <a href="#fr-1-1">↩</a></p> |
| </li> |
| <li id="footnote-2"> |
| <p>FIXME: why do we check whether the expected type is rigid for this. <a href="#fr-2-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h2 id="significant-changes-and-quirks"><a class="header" href="#significant-changes-and-quirks">Significant changes and quirks</a></h2> |
| <p>While some of the items below are already mentioned separately, this page tracks the |
| main changes from the old trait system implementation. This also mentions some ways |
| in which the solver significantly diverges from an idealized implementation. This |
| document simplifies and ignores edge cases. It is recommended to add an implicit |
| "mostly" to each statement.</p> |
| <h3 id="canonicalization-2"><a class="header" href="#canonicalization-2">Canonicalization</a></h3> |
| <p>The new solver uses <a href="solve/./canonicalization.html">canonicalization</a> when evaluating nested goals. In case there |
| are possibly multiple candidates, each candidate is eagerly canonicalized. We then |
| attempt to merge their canonical responses. This differs from the old implementation |
| which does not use canonicalization inside of the trait system.</p> |
| <p>This has a some major impacts on the design of both solvers. Without using |
| canonicalization to stash the constraints of candidates, candidate selection has |
| to discard the constraints of each candidate, only applying the constraints by |
| reevaluating the candidate after it has been selected: <a href="https://github.com/rust-lang/rust/blob/47dd709bedda8127e8daec33327e0a9d0cdae845/compiler/rustc_trait_selection/src/traits/select/mod.rs#L1232-L1237">source</a>. |
| Without canonicalization it is also not possible to cache the inference constraints |
| from evaluating a goal. This causes the old implementation to have two systems: |
| <em>evaluate</em> and <em>fulfill</em>. <em>Evaluation</em> is cached, does not apply inference constraints |
| and is used when selecting candidates. <em>Fulfillment</em> applies inference and region |
| constraints is not cached and applies inference constraints.</p> |
| <p>By using canonicalization, the new implementation is able to merge <em>evaluation</em> and |
| <em>fulfillment</em>, avoiding complexity and subtle differences in behavior. It greatly |
| simplifies caching and prevents accidentally relying on untracked information. |
| It allows us to avoid reevaluating candidates after selection and enables us to merge |
| the responses of multiple candidates. However, canonicalizing goals during evaluation |
| forces the new implementation to use a fixpoint algorithm when encountering cycles |
| during trait solving: <a href="https://github.com/rust-lang/rust/blob/df8ac8f1d74cffb96a93ae702d16e224f5b9ee8c/compiler/rustc_trait_selection/src/solve/search_graph.rs#L382-L387">source</a>.</p> |
| <h3 id="deferred-alias-equality"><a class="header" href="#deferred-alias-equality">Deferred alias equality</a></h3> |
| <p>The new implementation emits <code>AliasRelate</code> goals when relating aliases while the |
| old implementation structurally relates the aliases instead. This enables the |
| new solver to stall equality until it is able to normalize the related aliases.</p> |
| <p>The behavior of the old solver is incomplete and relies on eager normalization |
| which replaces ambiguous aliases with inference variables. As this is |
| not possible for aliases containing bound variables, the old implementation does |
| not handle aliases inside of binders correctly, e.g. <a href="https://github.com/rust-lang/rust/issues/102048">#102048</a>. See the chapter on |
| <a href="solve/../normalization.html">normalization</a> for more details.</p> |
| <h3 id="eagerly-evaluating-nested-goals"><a class="header" href="#eagerly-evaluating-nested-goals">Eagerly evaluating nested goals</a></h3> |
| <p>The new implementation eagerly handles nested goals instead of returning |
| them to the caller. The old implementation does both. In evaluation nested |
| goals <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_trait_selection/src/traits/select/mod.rs#L1271-L1277">are eagerly handled</a>, while fulfillment simply |
| <a href="https://github.com/rust-lang/rust/blob/df8ac8f1d74cffb96a93ae702d16e224f5b9ee8c/compiler/rustc_trait_selection/src/traits/fulfill.rs#L708-L712">returns them for later processing</a>.</p> |
| <p>As the new implementation has to be able to eagerly handle nested goals for |
| candidate selection, always doing so reduces complexity. It may also enable |
| us to merge more candidates in the future.</p> |
| <h3 id="nested-goals-are-evaluated-until-reaching-a-fixpoint"><a class="header" href="#nested-goals-are-evaluated-until-reaching-a-fixpoint">Nested goals are evaluated until reaching a fixpoint</a></h3> |
| <p>The new implementation always evaluates goals in a loop until reaching a fixpoint. |
| The old implementation only does so in <em>fulfillment</em>, but not in <em>evaluation</em>. |
| Always doing so strengthens inference and is reduces the order dependence of |
| the trait solver. See <a href="https://github.com/rust-lang/trait-system-refactor-initiative/issues/102">trait-system-refactor-initiative#102</a>.</p> |
| <h3 id="proof-trees-and-providing-diagnostics-information"><a class="header" href="#proof-trees-and-providing-diagnostics-information">Proof trees and providing diagnostics information</a></h3> |
| <p>The new implementation does not track diagnostics information directly, |
| instead providing <a href="solve/./proof-trees.html">proof trees</a> which are used to lazily compute the |
| relevant information. This is not yet fully fleshed out and somewhat hacky. |
| The goal is to avoid tracking this information in the happy path to improve |
| performance and to avoid accidentally relying on diagnostics data for behavior.</p> |
| <h2 id="major-quirks-of-the-new-implementation"><a class="header" href="#major-quirks-of-the-new-implementation">Major quirks of the new implementation</a></h2> |
| <h3 id="hiding-impls-if-there-are-any-env-candidates"><a class="header" href="#hiding-impls-if-there-are-any-env-candidates">Hiding impls if there are any env candidates</a></h3> |
| <p>If there is at least one <code>ParamEnv</code> or <code>AliasBound</code> candidate to prove |
| some <code>Trait</code> goal, we discard all impl candidates for both <code>Trait</code> and |
| <code>Projection</code> goals: <a href="https://github.com/rust-lang/rust/blob/03994e498df79aa1f97f7bbcfd52d57c8e865049/compiler/rustc_trait_selection/src/solve/assembly/mod.rs#L785-L789">source</a>. This prevents users from |
| using an impl which is entirely covered by a <code>where</code>-bound, matching the |
| behavior of the old implementation and avoiding some weird errors, |
| e.g. <a href="https://github.com/rust-lang/trait-system-refactor-initiative/issues/76">trait-system-refactor-initiative#76</a>.</p> |
| <h3 id="normalizesto-goals-are-a-function"><a class="header" href="#normalizesto-goals-are-a-function"><code>NormalizesTo</code> goals are a function</a></h3> |
| <p>See the <a href="solve/../normalization.html">normalization</a> chapter. We replace the expected term with an unconstrained |
| inference variable before computing <code>NormalizesTo</code> goals to prevent it from affecting |
| normalization. This means that <code>NormalizesTo</code> goals are handled somewhat differently |
| from all other goal kinds and need some additional solver support. Most notably, |
| their ambiguous nested goals are returned to the caller which then evaluates them. |
| See <a href="https://github.com/rust-lang/rust/pull/122687">#122687</a> for more details.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="coerceunsized"><a class="header" href="#coerceunsized"><a href="https://doc.rust-lang.org/std/ops/trait.CoerceUnsized.html"><code>CoerceUnsized</code></a></a></h1> |
| <p><code>CoerceUnsized</code> is primarily concerned with data containers. When a struct |
| (typically, a smart pointer) implements <code>CoerceUnsized</code>, that means that the |
| data it points to is being unsized.</p> |
| <p>Some implementors of <code>CoerceUnsized</code> include:</p> |
| <ul> |
| <li><code>&T</code></li> |
| <li><code>Arc<T></code></li> |
| <li><code>Box<T></code></li> |
| </ul> |
| <p>This trait is (eventually) intended to be implemented by user-written smart |
| pointers, and there are rules about when a type is allowed to implement |
| <code>CoerceUnsized</code> that are explained in the trait's documentation.</p> |
| <h1 id="unsize"><a class="header" href="#unsize"><a href="https://doc.rust-lang.org/std/marker/trait.Unsize.html"><code>Unsize</code></a></a></h1> |
| <p>To contrast, the <code>Unsize</code> trait is concerned the actual types that are allowed |
| to be unsized.</p> |
| <p>This is not intended to be implemented by users ever, since <code>Unsize</code> does not |
| instruct the compiler (namely codegen) <em>how</em> to unsize a type, just whether it |
| is allowed to be unsized. This is paired somewhat intimately with codegen |
| which must understand how types are represented and unsized.</p> |
| <h2 id="primitive-unsizing-implementations"><a class="header" href="#primitive-unsizing-implementations">Primitive unsizing implementations</a></h2> |
| <p>Built-in implementations are provided for:</p> |
| <ul> |
| <li><code>T</code> -> <code>dyn Trait + 'a</code> when <code>T: Trait</code> (and <code>T: Sized + 'a</code>, and <code>Trait</code> |
| is dyn-compatible<sup class="footnote-reference" id="fr-2-1"><a href="#footnote-2">1</a></sup>).</li> |
| <li><code>[T; N]</code> -> <code>[T]</code></li> |
| </ul> |
| <h2 id="structural-implementations"><a class="header" href="#structural-implementations">Structural implementations</a></h2> |
| <p>There is one implementation of <code>Unsize</code> which can be thought of as |
| structural:</p> |
| <ul> |
| <li><code>Struct<.., Pi, .., Pj, ..>: Unsize<Struct<.., Ui, .., Uj, ..>></code> given |
| <code>TailField<Pi, .., Pj>: Unsize<Ui, .. Uj></code>, which allows the tail field of a |
| struct to be unsized if it is the only field that mentions generic parameters |
| <code>Pi</code>, .., <code>Pj</code> (which don't need to be contiguous).</li> |
| </ul> |
| <p>The rules for struct unsizing are slightly complicated, since they |
| may allow more than one parameter to be changed (not necessarily unsized) and |
| are best stated in terms of the tail field of the struct.</p> |
| <p>(Tuple unsizing was previously implemented behind the feature gate |
| <code>unsized_tuple_coercion</code>, but the implementation was removed by <a href="https://github.com/rust-lang/rust/pull/137728">#137728</a>.)</p> |
| <h2 id="upcasting-implementations"><a class="header" href="#upcasting-implementations">Upcasting implementations</a></h2> |
| <p>Two things are called "upcasting" internally:</p> |
| <ol> |
| <li>True upcasting <code>dyn SubTrait</code> -> <code>dyn SuperTrait</code> (this also allows |
| dropping auto traits and adjusting lifetimes, as below).</li> |
| <li>Dropping auto traits and adjusting the lifetimes of dyn trait |
| <em>without changing the principal<sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">2</a></sup></em>: |
| <code>dyn Trait + AutoTraits... + 'a</code> -> <code>dyn Trait + NewAutoTraits... + 'b</code> |
| when <code>AutoTraits</code> ⊇ <code>NewAutoTraits</code>, and <code>'a: 'b</code>.</li> |
| </ol> |
| <p>These may seem like different operations, since (1.) includes adjusting the |
| vtable of a dyn trait, while (2.) is a no-op. However, to the type system, |
| these are handled with much the same code.</p> |
| <p>This built-in implementation of <code>Unsize</code> is the most involved, particularly |
| after <a href="https://github.com/rust-lang/rust/pull/114036">it was reworked</a> to |
| support the complexities of associated types.</p> |
| <p>Specifically, the upcasting algorithm involves: For each supertrait of the |
| source dyn trait's principal (including itself)...</p> |
| <ol> |
| <li>Unify the super trait ref with the principal of the target (making sure |
| we only ever upcast to a true supertrait, and never <a href="https://github.com/rust-lang/rust/blob/f3457dbf84cd86d284454d12705861398ece76c3/tests/ui/traits/trait-upcasting/illegal-upcast-from-impl.rs#L19">via an impl</a>).</li> |
| <li>For every auto trait in the target, check that it's present in the source |
| (allowing us to drop auto traits, but never gain new ones).</li> |
| <li>For every projection in the target, check that it unifies with a single |
| projection in the source (since there may be more than one given |
| <code>trait Sub: Sup<.., A = i32> + Sup<.., A = u32></code>).</li> |
| </ol> |
| <p>Specifically, (3.) prevents a choice of projection bound to guide inference |
| unnecessarily, though it may guide inference when it is unambiguous.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-2"> |
| <p>Formerly known as "object safe". <a href="#fr-2-1">↩</a></p> |
| </li> |
| <li id="footnote-1"> |
| <p>The principal is the one non-auto trait of a <code>dyn Trait</code>. <a href="#fr-1-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="type-checking-1"><a class="header" href="#type-checking-1">Type checking</a></h1> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/index.html"><code>hir_analysis</code></a> crate contains the source for "type collection" as well |
| as a bunch of related functionality. |
| Checking the bodies of functions is implemented in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/index.html"><code>hir_typeck</code></a> crate. |
| These crates draw heavily on the <a href="./type-inference.html">type inference</a> and <a href="./traits/resolution.html">trait solving</a>.</p> |
| <h2 id="type-collection"><a class="header" href="#type-collection">Type collection</a></h2> |
| <p>Type "collection" is the process of converting the types found in the HIR |
| (<code>hir::Ty</code>), which represent the syntactic things that the user wrote, into the |
| <strong>internal representation</strong> used by the compiler (<code>Ty<'tcx></code>) – we also do |
| similar conversions for where-clauses and other bits of the function signature.</p> |
| <p>To try and get a sense of the difference, consider this function:</p> |
| <pre><code class="language-rust ignore">struct Foo { } |
| fn foo(x: Foo, y: self::Foo) { ... } |
| // ^^^ ^^^^^^^^^</code></pre> |
| <p>Those two parameters <code>x</code> and <code>y</code> each have the same type: but they will have |
| distinct <code>hir::Ty</code> nodes. Those nodes will have different spans, and of course |
| they encode the path somewhat differently. But once they are "collected" into |
| <code>Ty<'tcx></code> nodes, they will be represented by the exact same internal type.</p> |
| <p>Collection is defined as a bundle of <a href="./query.html">queries</a> for computing information about |
| the various functions, traits, and other items in the crate being compiled. |
| Note that each of these queries is concerned with <em>interprocedural</em> things – |
| for example, for a function definition, collection will figure out the type and |
| signature of the function, but it will not visit the <em>body</em> of the function in |
| any way, nor examine type annotations on local variables (that's the job of |
| type <em>checking</em>).</p> |
| <p>For more details, see the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/collect/index.html"><code>collect</code></a> module.</p> |
| <p><strong>TODO</strong>: actually talk about type checking... <a href="https://github.com/rust-lang/rustc-dev-guide/issues/1161">#1161</a></p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="method-lookup"><a class="header" href="#method-lookup">Method lookup</a></h1> |
| <p>Method lookup can be rather complex due to the interaction of a number |
| of factors, such as self types, autoderef, trait lookup, etc. This |
| file provides an overview of the process. More detailed notes are in |
| the code itself, naturally.</p> |
| <p>One way to think of method lookup is that we convert an expression of |
| the form <code>receiver.method(...)</code> into a more explicit <a href="https://doc.rust-lang.org/nightly/book/ch19-03-advanced-traits.html#fully-qualified-syntax-for-disambiguation-calling-methods-with-the-same-name">fully-qualified syntax</a> |
| (formerly called <a href="https://github.com/rust-lang/rfcs/blob/master/text/0132-ufcs.md">UFCS</a>):</p> |
| <ul> |
| <li><code>Trait::method(ADJ(receiver), ...)</code> for a trait call</li> |
| <li><code>ReceiverType::method(ADJ(receiver), ...)</code> for an inherent method call</li> |
| </ul> |
| <p>Here <code>ADJ</code> is some kind of adjustment, which is typically a series of |
| autoderefs and then possibly an autoref (e.g., <code>&**receiver</code>). However |
| we sometimes do other adjustments and coercions along the way, in |
| particular unsizing (e.g., converting from <code>[T; n]</code> to <code>[T]</code>).</p> |
| <p>Method lookup is divided into two major phases:</p> |
| <ol> |
| <li>Probing (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/method/probe/"><code>probe.rs</code></a>). The probe phase is when we decide what method |
| to call and how to adjust the receiver.</li> |
| <li>Confirmation (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/method/confirm/"><code>confirm.rs</code></a>). The confirmation phase "applies" |
| this selection, updating the side-tables, unifying type variables, and |
| otherwise doing side-effectful things.</li> |
| </ol> |
| <p>One reason for this division is to be more amenable to caching. The |
| probe phase produces a "pick" (<code>probe::Pick</code>), which is designed to be |
| cacheable across method-call sites. Therefore, it does not include |
| inference variables or other information.</p> |
| <h2 id="the-probe-phase"><a class="header" href="#the-probe-phase">The Probe phase</a></h2> |
| <h3 id="steps"><a class="header" href="#steps">Steps</a></h3> |
| <p>The first thing that the probe phase does is to create a series of |
| <em>steps</em>. This is done by progressively dereferencing the receiver type |
| until it cannot be deref'd anymore, as well as applying an optional |
| "unsize" step. So if the receiver has type <code>Rc<Box<[T; 3]>></code>, this |
| might yield:</p> |
| <ol> |
| <li><code>Rc<Box<[T; 3]>></code></li> |
| <li><code>Box<[T; 3]></code></li> |
| <li><code>[T; 3]</code></li> |
| <li><code>[T]</code></li> |
| </ol> |
| <h3 id="candidate-assembly-1"><a class="header" href="#candidate-assembly-1">Candidate assembly</a></h3> |
| <p>We then search along those steps to create a list of <em>candidates</em>. A |
| <code>Candidate</code> is a method item that might plausibly be the method being |
| invoked. For each candidate, we'll derive a "transformed self type" |
| that takes into account explicit self.</p> |
| <p>Candidates are grouped into two kinds, inherent and extension.</p> |
| <p><strong>Inherent candidates</strong> are those that are derived from the |
| type of the receiver itself. So, if you have a receiver of some |
| nominal type <code>Foo</code> (e.g., a struct), any methods defined within an |
| impl like <code>impl Foo</code> are inherent methods. Nothing needs to be |
| imported to use an inherent method, they are associated with the type |
| itself (note that inherent impls can only be defined in the same |
| crate as the type itself).</p> |
| <!-- |
| FIXME: Inherent candidates are not always derived from impls. If you |
| have a trait object, such as a value of type `Box<ToString>`, then the |
| trait methods (`to_string()`, in this case) are inherently associated |
| with it. Another case is type parameters, in which case the methods of |
| their bounds are inherent. However, this part of the rules is subject |
| to change: when DST's "impl Trait for Trait" is complete, trait object |
| dispatch could be subsumed into trait matching, and the type parameter |
| behavior should be reconsidered in light of where clauses. |
| |
| Is this still accurate? |
| --> |
| <p><strong>Extension candidates</strong> are derived from imported traits. If I have |
| the trait <code>ToString</code> imported, and I call <code>to_string()</code> as a method, |
| then we will list the <code>to_string()</code> definition in each impl of |
| <code>ToString</code> as a candidate. These kinds of method calls are called |
| "extension methods".</p> |
| <p>So, let's continue our example. Imagine that we were calling a method |
| <code>foo</code> with the receiver <code>Rc<Box<[T; 3]>></code> and there is a trait <code>Foo</code> |
| that defines it with <code>&self</code> for the type <code>Rc<U></code> as well as a method |
| on the type <code>Box</code> that defines <code>foo</code> but with <code>&mut self</code>. Then we |
| might have two candidates:</p> |
| <ul> |
| <li><code>&Rc<U></code> as an extension candidate</li> |
| <li><code>&mut Box<U></code> as an inherent candidate</li> |
| </ul> |
| <h3 id="candidate-search"><a class="header" href="#candidate-search">Candidate search</a></h3> |
| <p>Finally, to actually pick the method, we will search down the steps, |
| trying to match the receiver type against the candidate types. At |
| each step, we also consider an auto-ref and auto-mut-ref to see whether |
| that makes any of the candidates match. For each resulting receiver |
| type, we consider inherent candidates before extension candidates. |
| If there are multiple matching candidates in a group, we report an |
| error, except that multiple impls of the same trait are treated as a |
| single match. Otherwise we pick the first match we find.</p> |
| <p>In the case of our example, the first step is <code>Rc<Box<[T; 3]>></code>, |
| which does not itself match any candidate. But when we autoref it, we |
| get the type <code>&Rc<Box<[T; 3]>></code> which matches <code>&Rc<U></code>. We would then |
| recursively consider all where-clauses that appear on the impl: if |
| those match (or we cannot rule out that they do), then this is the |
| method we would pick. Otherwise, we would continue down the series of |
| steps.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="variance-of-type-and-lifetime-parameters"><a class="header" href="#variance-of-type-and-lifetime-parameters">Variance of type and lifetime parameters</a></h1> |
| <p>For a more general background on variance, see the <a href="./appendix/background.html">background</a> appendix.</p> |
| <p>During type checking we must infer the variance of type and lifetime |
| parameters. The algorithm is taken from Section 4 of the paper <a href="https://people.cs.umass.edu/~yannis/variance-extended2011.pdf">"Taming the |
| Wildcards: Combining Definition- and Use-Site Variance"</a> published in |
| PLDI'11 and written by Altidor et al., and hereafter referred to as The Paper.</p> |
| <p>This inference is explicitly designed <em>not</em> to consider the uses of |
| types within code. To determine the variance of type parameters |
| defined on type <code>X</code>, we only consider the definition of the type <code>X</code> |
| and the definitions of any types it references.</p> |
| <p>We only infer variance for type parameters found on <em>data types</em> |
| like structs and enums. In these cases, there is a fairly straightforward |
| explanation for what variance means. The variance of the type |
| or lifetime parameters defines whether <code>T<A></code> is a subtype of <code>T<B></code> |
| (resp. <code>T<'a></code> and <code>T<'b></code>) based on the relationship of <code>A</code> and <code>B</code> |
| (resp. <code>'a</code> and <code>'b</code>).</p> |
| <p>We do not infer variance for type parameters found on traits, functions, |
| or impls. Variance on trait parameters can indeed make sense |
| (and we used to compute it) but it is actually rather subtle in |
| meaning and not that useful in practice, so we removed it. See the |
| <a href="variance.html#addendum">addendum</a> for some details. Variances on function/impl parameters, on the |
| other hand, doesn't make sense because these parameters are instantiated and |
| then forgotten, they don't persist in types or compiled byproducts.</p> |
| <blockquote> |
| <p><strong>Notation</strong></p> |
| <p>We use the notation of The Paper throughout this chapter:</p> |
| <ul> |
| <li><code>+</code> is <em>covariance</em>.</li> |
| <li><code>-</code> is <em>contravariance</em>.</li> |
| <li><code>*</code> is <em>bivariance</em>.</li> |
| <li><code>o</code> is <em>invariance</em>.</li> |
| </ul> |
| </blockquote> |
| <h2 id="the-algorithm"><a class="header" href="#the-algorithm">The algorithm</a></h2> |
| <p>The basic idea is quite straightforward. We iterate over the types |
| defined and, for each use of a type parameter <code>X</code>, accumulate a |
| constraint indicating that the variance of <code>X</code> must be valid for the |
| variance of that use site. We then iteratively refine the variance of |
| <code>X</code> until all constraints are met. There is <em>always</em> a solution, because at |
| the limit we can declare all type parameters to be invariant and all |
| constraints will be satisfied.</p> |
| <p>As a simple example, consider:</p> |
| <pre><code class="language-rust ignore">enum Option<A> { Some(A), None } |
| enum OptionalFn<B> { Some(|B|), None } |
| enum OptionalMap<C> { Some(|C| -> C), None }</code></pre> |
| <p>Here, we will generate the constraints:</p> |
| <pre><code class="language-text">1. V(A) <= + |
| 2. V(B) <= - |
| 3. V(C) <= + |
| 4. V(C) <= - |
| </code></pre> |
| <p>These indicate that (1) the variance of A must be at most covariant; |
| (2) the variance of B must be at most contravariant; and (3, 4) the |
| variance of C must be at most covariant <em>and</em> contravariant. All of these |
| results are based on a variance lattice defined as follows:</p> |
| <pre><code class="language-text"> * Top (bivariant) |
| - + |
| o Bottom (invariant) |
| </code></pre> |
| <p>Based on this lattice, the solution <code>V(A)=+</code>, <code>V(B)=-</code>, <code>V(C)=o</code> is the |
| optimal solution. Note that there is always a naive solution which |
| just declares all variables to be invariant.</p> |
| <p>You may be wondering why fixed-point iteration is required. The reason |
| is that the variance of a use site may itself be a function of the |
| variance of other type parameters. In full generality, our constraints |
| take the form:</p> |
| <pre><code class="language-text">V(X) <= Term |
| Term := + | - | * | o | V(X) | Term x Term |
| </code></pre> |
| <p>Here the notation <code>V(X)</code> indicates the variance of a type/region |
| parameter <code>X</code> with respect to its defining class. <code>Term x Term</code> |
| represents the "variance transform" as defined in the paper:</p> |
| <blockquote> |
| <p>If the variance of a type variable <code>X</code> in type expression <code>E</code> is <code>V2</code> |
| and the definition-site variance of the corresponding type parameter |
| of a class <code>C</code> is <code>V1</code>, then the variance of <code>X</code> in the type expression |
| <code>C<E></code> is <code>V3 = V1.xform(V2)</code>.</p> |
| </blockquote> |
| <h2 id="constraints-1"><a class="header" href="#constraints-1">Constraints</a></h2> |
| <p>If I have a struct or enum with where clauses:</p> |
| <pre><code class="language-rust ignore">struct Foo<T: Bar> { ... }</code></pre> |
| <p>you might wonder whether the variance of <code>T</code> with respect to <code>Bar</code> affects the |
| variance <code>T</code> with respect to <code>Foo</code>. I claim no. The reason: assume that <code>T</code> is |
| invariant with respect to <code>Bar</code> but covariant with respect to <code>Foo</code>. And then |
| we have a <code>Foo<X></code> that is upcast to <code>Foo<Y></code>, where <code>X <: Y</code>. However, while |
| <code>X : Bar</code>, <code>Y : Bar</code> does not hold. In that case, the upcast will be illegal, |
| but not because of a variance failure, but rather because the target type |
| <code>Foo<Y></code> is itself just not well-formed. Basically we get to assume |
| well-formedness of all types involved before considering variance.</p> |
| <h3 id="dependency-graph-management"><a class="header" href="#dependency-graph-management">Dependency graph management</a></h3> |
| <p>Because variance is a whole-crate inference, its dependency graph |
| can become quite muddled if we are not careful. To resolve this, we refactor |
| into two queries:</p> |
| <ul> |
| <li><code>crate_variances</code> computes the variance for all items in the current crate.</li> |
| <li><code>variances_of</code> accesses the variance for an individual reading; it |
| works by requesting <code>crate_variances</code> and extracting the relevant data.</li> |
| </ul> |
| <p>If you limit yourself to reading <code>variances_of</code>, your code will only |
| depend then on the inference of that particular item.</p> |
| <p>Ultimately, this setup relies on the <a href="./queries/incremental-compilation.html">red-green algorithm</a>. In particular, |
| every variance query effectively depends on all type definitions in the entire |
| crate (through <code>crate_variances</code>), but since most changes will not result in a |
| change to the actual results from variance inference, the <code>variances_of</code> query |
| will wind up being considered green after it is re-evaluated.</p> |
| <p><a id="addendum"></a></p> |
| <h2 id="addendum-variance-on-traits"><a class="header" href="#addendum-variance-on-traits">Addendum: Variance on traits</a></h2> |
| <p>As mentioned above, we used to permit variance on traits. This was |
| computed based on the appearance of trait type parameters in |
| method signatures and was used to represent the compatibility of |
| vtables in trait objects (and also "virtual" vtables or dictionary |
| in trait bounds). One complication was that variance for |
| associated types is less obvious, since they can be projected out |
| and put to myriad uses, so it's not clear when it is safe to allow |
| <code>X<A>::Bar</code> to vary (or indeed just what that means). Moreover (as |
| covered below) all inputs on any trait with an associated type had |
| to be invariant, limiting the applicability. Finally, the |
| annotations (<code>MarkerTrait</code>, <code>PhantomFn</code>) needed to ensure that all |
| trait type parameters had a variance were confusing and annoying |
| for little benefit.</p> |
| <p>Just for historical reference, I am going to preserve some text indicating how |
| one could interpret variance and trait matching.</p> |
| <h3 id="variance-and-object-types"><a class="header" href="#variance-and-object-types">Variance and object types</a></h3> |
| <p>Just as with structs and enums, we can decide the subtyping |
| relationship between two object types <code>&Trait<A></code> and <code>&Trait<B></code> |
| based on the relationship of <code>A</code> and <code>B</code>. Note that for object |
| types we ignore the <code>Self</code> type parameter – it is unknown, and |
| the nature of dynamic dispatch ensures that we will always call a |
| function that is expected the appropriate <code>Self</code> type. However, we |
| must be careful with the other type parameters, or else we could |
| end up calling a function that is expecting one type but provided |
| another.</p> |
| <p>To see what I mean, consider a trait like so:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait ConvertTo<A> { |
| fn convertTo(&self) -> A; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Intuitively, If we had one object <code>O=&ConvertTo<Object></code> and another |
| <code>S=&ConvertTo<String></code>, then <code>S <: O</code> because <code>String <: Object</code> |
| (presuming Java-like "string" and "object" types, my go to examples |
| for subtyping). The actual algorithm would be to compare the |
| (explicit) type parameters pairwise respecting their variance: here, |
| the type parameter A is covariant (it appears only in a return |
| position), and hence we require that <code>String <: Object</code>.</p> |
| <p>You'll note though that we did not consider the binding for the |
| (implicit) <code>Self</code> type parameter: in fact, it is unknown, so that's |
| good. The reason we can ignore that parameter is precisely because we |
| don't need to know its value until a call occurs, and at that time (as |
| you said) the dynamic nature of virtual dispatch means the code we run |
| will be correct for whatever value <code>Self</code> happens to be bound to for |
| the particular object whose method we called. <code>Self</code> is thus different |
| from <code>A</code>, because the caller requires that <code>A</code> be known in order to |
| know the return type of the method <code>convertTo()</code>. (As an aside, we |
| have rules preventing methods where <code>Self</code> appears outside of the |
| receiver position from being called via an object.)</p> |
| <h3 id="trait-variance-and-vtable-resolution"><a class="header" href="#trait-variance-and-vtable-resolution">Trait variance and vtable resolution</a></h3> |
| <p>But traits aren't only used with objects. They're also used when |
| deciding whether a given impl satisfies a given trait bound. To set the |
| scene here, imagine I had a function:</p> |
| <pre><code class="language-rust ignore">fn convertAll<A,T:ConvertTo<A>>(v: &[T]) { ... }</code></pre> |
| <p>Now imagine that I have an implementation of <code>ConvertTo</code> for <code>Object</code>:</p> |
| <pre><code class="language-rust ignore">impl ConvertTo<i32> for Object { ... }</code></pre> |
| <p>And I want to call <code>convertAll</code> on an array of strings. Suppose |
| further that for whatever reason I specifically supply the value of |
| <code>String</code> for the type parameter <code>T</code>:</p> |
| <pre><code class="language-rust ignore">let mut vector = vec!["string", ...]; |
| convertAll::<i32, String>(vector);</code></pre> |
| <p>Is this legal? To put another way, can we apply the <code>impl</code> for |
| <code>Object</code> to the type <code>String</code>? The answer is yes, but to see why |
| we have to expand out what will happen:</p> |
| <ul> |
| <li> |
| <p><code>convertAll</code> will create a pointer to one of the entries in the |
| vector, which will have type <code>&String</code></p> |
| </li> |
| <li> |
| <p>It will then call the impl of <code>convertTo()</code> that is intended |
| for use with objects. This has the type <code>fn(self: &Object) -> i32</code>.</p> |
| <p>It is OK to provide a value for <code>self</code> of type <code>&String</code> because |
| <code>&String <: &Object</code>.</p> |
| </li> |
| </ul> |
| <p>OK, so intuitively we want this to be legal, so let's bring this back |
| to variance and see whether we are computing the correct result. We |
| must first figure out how to phrase the question "is an impl for |
| <code>Object,i32</code> usable where an impl for <code>String,i32</code> is expected?"</p> |
| <p>Maybe it's helpful to think of a dictionary-passing implementation of |
| type classes. In that case, <code>convertAll()</code> takes an implicit parameter |
| representing the impl. In short, we <em>have</em> an impl of type:</p> |
| <pre><code class="language-text">V_O = ConvertTo<i32> for Object |
| </code></pre> |
| <p>and the function prototype expects an impl of type:</p> |
| <pre><code class="language-text">V_S = ConvertTo<i32> for String |
| </code></pre> |
| <p>As with any argument, this is legal if the type of the value given |
| (<code>V_O</code>) is a subtype of the type expected (<code>V_S</code>). So is <code>V_O <: V_S</code>? |
| The answer will depend on the variance of the various parameters. In |
| this case, because the <code>Self</code> parameter is contravariant and <code>A</code> is |
| covariant, it means that:</p> |
| <pre><code class="language-text">V_O <: V_S iff |
| i32 <: i32 |
| String <: Object |
| </code></pre> |
| <p>These conditions are satisfied and so we are happy.</p> |
| <h3 id="variance-and-associated-types"><a class="header" href="#variance-and-associated-types">Variance and associated types</a></h3> |
| <p>Traits with associated types – or at minimum projection |
| expressions – must be invariant with respect to all of their |
| inputs. To see why this makes sense, consider what subtyping for a |
| trait reference means:</p> |
| <pre><code class="language-text"><T as Trait> <: <U as Trait> |
| </code></pre> |
| <p>means that if I know that <code>T as Trait</code>, I also know that <code>U as Trait</code>. Moreover, if you think of it as dictionary passing style, |
| it means that a dictionary for <code><T as Trait></code> is safe to use where |
| a dictionary for <code><U as Trait></code> is expected.</p> |
| <p>The problem is that when you can project types out from <code><T as Trait></code>, the relationship to types projected out of <code><U as Trait></code> |
| is completely unknown unless <code>T==U</code> (see #21726 for more |
| details). Making <code>Trait</code> invariant ensures that this is true.</p> |
| <p>Another related reason is that if we didn't make traits with |
| associated types invariant, then projection is no longer a |
| function with a single result. Consider:</p> |
| <pre><code class="language-rust ignore">trait Identity { type Out; fn foo(&self); } |
| impl<T> Identity for T { type Out = T; ... }</code></pre> |
| <p>Now if I have <code><&'static () as Identity>::Out</code>, this can be |
| validly derived as <code>&'a ()</code> for any <code>'a</code>:</p> |
| <pre><code class="language-text"><&'a () as Identity> <: <&'static () as Identity> |
| if &'static () < : &'a () -- Identity is contravariant in Self |
| if 'static : 'a -- Subtyping rules for relations |
| </code></pre> |
| <p>This change otoh means that <code><'static () as Identity>::Out</code> is |
| always <code>&'static ()</code> (which might then be upcast to <code>'a ()</code>, |
| separately). This was helpful in solving #21750.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="coherence"><a class="header" href="#coherence">Coherence</a></h1> |
| <blockquote> |
| <p>NOTE: this is based on <a href="https://github.com/rust-lang/rust/pull/121848">notes by @lcnr</a></p> |
| </blockquote> |
| <p>Coherence checking is what detects both of trait impls and inherent impls overlapping with others. |
| (reminder: <a href="https://doc.rust-lang.org/reference/items/implementations.html#inherent-implementations">inherent impls</a> are impls of concrete types like <code>impl MyStruct {}</code>)</p> |
| <p>Overlapping trait impls always produce an error, |
| while overlapping inherent impls result in an error only if they have methods with the same name.</p> |
| <p>Checking for overlaps is split in two parts. First there's the <a href="coherence.html#overlap-checks">overlap check(s)</a>, |
| which finds overlaps between traits and inherent implementations that the compiler currently knows about.</p> |
| <p>However, Coherence also results in an error if any other impls <strong>could</strong> exist, |
| even if they are currently unknown. |
| This affects impls which may get added to upstream crates in a backwards compatible way, |
| and impls from downstream crates. |
| This is called the Orphan check.</p> |
| <h2 id="overlap-checks"><a class="header" href="#overlap-checks">Overlap checks</a></h2> |
| <p>Overlap checks are performed for both inherent impls, and for trait impls. |
| This uses the same overlap checking code, really done as two separate analyses. |
| Overlap checks always consider pairs of implementations, comparing them to each other.</p> |
| <p>Overlap checking for inherent impl blocks is done through <code>fn check_item</code> (in coherence/inherent_impls_overlap.rs), |
| where you can very clearly see that (at least for small <code>n</code>), the check really performs <code>n^2</code> |
| comparisons between impls.</p> |
| <p>In the case of traits, this check is currently done as part of building the <a href="traits/specialization.html">specialization graph</a>, |
| to handle specializing impls overlapping with their parent, but this may change in the future.</p> |
| <p>In both cases, all pairs of impls are checked for overlap.</p> |
| <p>Overlapping is sometimes partially allowed:</p> |
| <ol> |
| <li>for marker traits</li> |
| <li>under <a href="traits/specialization.html">specialization</a></li> |
| </ol> |
| <p>but normally isn't.</p> |
| <p>The overlap check has various modes (see <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_middle/traits/specialization_graph/enum.OverlapMode.html"><code>OverlapMode</code></a>). |
| Importantly, there's the explicit negative impl check, and the implicit negative impl check. |
| Both try to prove that an overlap is definitely impossible.</p> |
| <h3 id="the-explicit-negative-impl-check"><a class="header" href="#the-explicit-negative-impl-check">The explicit negative impl check</a></h3> |
| <p>This check is done in <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_trait_selection/traits/coherence/fn.impl_intersection_has_negative_obligation.html"><code>impl_intersection_has_negative_obligation</code></a>.</p> |
| <p>This check tries to find a negative trait implementation. |
| For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct MyCustomErrorType; |
| |
| // both in your own crate |
| impl From<&str> for MyCustomErrorType {} |
| impl<E> From<E> for MyCustomErrorType where E: Error {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example, we'd get: |
| <code>MyCustomErrorType: From<&str></code> and <code>MyCustomErrorType: From<?E></code>, giving <code>?E = &str</code>.</p> |
| <p>And thus, these two implementations would overlap. |
| However, libstd provides <code>&str: !Error</code>, and therefore guarantees that there |
| will never be a positive implementation of <code>&str: Error</code>, and thus there is no overlap.</p> |
| <p>Note that for this kind of negative impl check, we must have explicit negative implementations provided. |
| This is not currently stable.</p> |
| <h3 id="the-implicit-negative-impl-check"><a class="header" href="#the-implicit-negative-impl-check">The implicit negative impl check</a></h3> |
| <p>This check is done in <a href="https://doc.rust-lang.org/beta/nightly-rustc/rustc_trait_selection/traits/coherence/fn.impl_intersection_has_impossible_obligation.html"><code>impl_intersection_has_impossible_obligation</code></a>, |
| and does not rely on negative trait implementations and is stable.</p> |
| <p>Let's say there's a</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>impl From<MyLocalType> for Box<dyn Error> {} // in your own crate |
| impl<E> From<E> for Box<dyn Error> where E: Error {} // in std |
| <span class="boring">}</span></code></pre></pre> |
| <p>This would give: <code>Box<dyn Error>: From<MyLocalType></code>, and <code>Box<dyn Error>: From<?E></code>,<br /> |
| giving <code>?E = MyLocalType</code>.</p> |
| <p>In your crate there's no <code>MyLocalType: Error</code>, downstream crates cannot implement <code>Error</code> (a remote trait) for <code>MyLocalType</code> (a remote type). |
| Therefore, these two impls do not overlap. |
| Importantly, this works even if there isn't a <code>impl !Error for MyLocalType</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="opaque-types-type-alias-impl-trait"><a class="header" href="#opaque-types-type-alias-impl-trait">Opaque types (type alias <code>impl Trait</code>)</a></h1> |
| <p>Opaque types are syntax to declare an opaque type alias that only |
| exposes a specific set of traits as their interface; the concrete type in the |
| background is inferred from a certain set of use sites of the opaque type.</p> |
| <p>This is expressed by using <code>impl Trait</code> within type aliases, for example:</p> |
| <pre><code class="language-rust ignore">type Foo = impl Bar;</code></pre> |
| <p>This declares an opaque type named <code>Foo</code>, of which the only information is that |
| it implements <code>Bar</code>. Therefore, any of <code>Bar</code>'s interface can be used on a <code>Foo</code>, |
| but nothing else (regardless of whether it implements any other traits).</p> |
| <p>Since there needs to be a concrete background type, |
| you can (as of <!-- date-check --> January 2021) express that type |
| by using the opaque type in a "defining use site".</p> |
| <pre><code class="language-rust ignore">struct Struct; |
| impl Bar for Struct { /* stuff */ } |
| fn foo() -> Foo { |
| Struct |
| }</code></pre> |
| <p>Any other "defining use site" needs to produce the exact same type.</p> |
| <h2 id="defining-use-sites"><a class="header" href="#defining-use-sites">Defining use site(s)</a></h2> |
| <p>Currently only the return value of a function can be a defining use site |
| of an opaque type (and only if the return type of that function contains |
| the opaque type).</p> |
| <p>The defining use of an opaque type can be any code <em>within</em> the parent |
| of the opaque type definition. This includes any siblings of the |
| opaque type and all children of the siblings.</p> |
| <p>The initiative for <em>"not causing fatal brain damage to developers due to |
| accidentally running infinite loops in their brain while trying to |
| comprehend what the type system is doing"</em> has decided to disallow children |
| of opaque types to be defining use sites.</p> |
| <h3 id="associated-opaque-types"><a class="header" href="#associated-opaque-types">Associated opaque types</a></h3> |
| <p>Associated opaque types can be defined by any other associated item |
| on the same trait <code>impl</code> or a child of these associated items. For instance:</p> |
| <pre><code class="language-rust ignore">trait Baz { |
| type Foo; |
| fn foo() -> Self::Foo; |
| } |
| |
| struct Quux; |
| |
| impl Baz for Quux { |
| type Foo = impl Bar; |
| fn foo() -> Self::Foo { ... } |
| }</code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="inference-of-opaque-types-impl-trait"><a class="header" href="#inference-of-opaque-types-impl-trait">Inference of opaque types (<code>impl Trait</code>)</a></h1> |
| <p>This page describes how the compiler infers the <a href="./borrow_check/region_inference/member_constraints.html?highlight=%22hidden%20type%22#member-constraints">hidden type</a> for an <a href="./opaque-types-type-alias-impl-trait.html">opaque type</a>. |
| This kind of type inference is particularly complex because, |
| unlike other kinds of type inference, |
| it can work across functions and function bodies.</p> |
| <h2 id="running-example"><a class="header" href="#running-example">Running example</a></h2> |
| <p>To help explain how it works, let's consider an example.</p> |
| <pre><pre class="playground"><code class="language-rust">#![feature(type_alias_impl_trait)] |
| mod m { |
| pub type Seq<T> = impl IntoIterator<Item = T>; |
| |
| #[define_opaque(Seq)] |
| pub fn produce_singleton<T>(t: T) -> Seq<T> { |
| vec![t] |
| } |
| |
| #[define_opaque(Seq)] |
| pub fn produce_doubleton<T>(t: T, u: T) -> Seq<T> { |
| vec![t, u] |
| } |
| } |
| |
| fn is_send<T: Send>(_: &T) {} |
| |
| pub fn main() { |
| let elems = m::produce_singleton(22); |
| |
| is_send(&elems); |
| |
| for elem in elems { |
| println!("elem = {:?}", elem); |
| } |
| }</code></pre></pre> |
| <p>In this code, the <em>opaque type</em> is <code>Seq<T></code>. |
| Its defining scope is the module <code>m</code>. |
| Its <em>hidden type</em> is <code>Vec<T></code>, |
| which is inferred from <code>m::produce_singleton</code> and <code>m::produce_doubleton</code>.</p> |
| <p>In the <code>main</code> function, the opaque type is out of its defining scope. |
| When <code>main</code> calls <code>m::produce_singleton</code>, it gets back a reference to the opaque type <code>Seq<i32></code>. |
| The <code>is_send</code> call checks that <code>Seq<i32>: Send</code>. |
| <code>Send</code> is not listed amongst the bounds of the impl trait, |
| but because of auto-trait leakage, we are able to infer that it holds. |
| The <code>for</code> loop desugaring requires that <code>Seq<T>: IntoIterator</code>, |
| which is provable from the bounds declared on <code>Seq<T></code>.</p> |
| <h3 id="type-checking-main"><a class="header" href="#type-checking-main">Type-checking <code>main</code></a></h3> |
| <p>Let's start by looking what happens when we type-check <code>main</code>. |
| Initially we invoke <code>produce_singleton</code> and the return type is an opaque type |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.ItemKind.html#variant.OpaqueTy"><code>OpaqueTy</code></a>.</p> |
| <h4 id="type-checking-the-for-loop"><a class="header" href="#type-checking-the-for-loop">Type-checking the for loop</a></h4> |
| <p>The for loop desugars the <code>in elems</code> part to <code>IntoIterator::into_iter(elems)</code>. |
| <code>elems</code> is of type <code>Seq<T></code>, so the type checker registers a <code>Seq<T>: IntoIterator</code> obligation. |
| This obligation is trivially satisfied, |
| because <code>Seq<T></code> is an opaque type (<code>impl IntoIterator<Item = T></code>) that has a bound for the trait. |
| Similar to how a <code>U: Foo</code> where bound allows <code>U</code> to trivially satisfy <code>Foo</code>, |
| opaque types' bounds are available to the type checker and are used to fulfill obligations.</p> |
| <p>The type of <code>elem</code> in the for loop is inferred to be <code><Seq<T> as IntoIterator>::Item</code>, which is <code>T</code>. |
| At no point is the type checker interested in the hidden type.</p> |
| <h4 id="type-checking-the-is_send-call"><a class="header" href="#type-checking-the-is_send-call">Type-checking the <code>is_send</code> call</a></h4> |
| <p>When trying to prove auto trait bounds, |
| we first repeat the process as above, |
| to see if the auto trait is in the bound list of the opaque type. |
| If that fails, we reveal the hidden type of the opaque type, |
| but only to prove this specific trait bound, not in general. |
| Revealing is done by invoking the <code>type_of</code> query on the <code>DefId</code> of the opaque type. |
| The query will internally request the hidden types from the defining function(s) |
| and return that (see <a href="opaque-types-impl-trait-inference.html#within-the-type_of-query">the section on <code>type_of</code></a> for more details).</p> |
| <h4 id="flowchart-of-type-checking-steps"><a class="header" href="#flowchart-of-type-checking-steps">Flowchart of type checking steps</a></h4> |
| <pre class="mermaid">flowchart TD |
| TypeChecking["type checking `main`"] |
| subgraph TypeOfSeq["type_of(Seq<T>) query"] |
| WalkModuleHir["Walk the HIR for the module `m`\nto find the hidden types from each\nfunction/const/static within"] |
| VisitProduceSingleton["visit `produce_singleton`"] |
| InterimType["`produce_singleton` hidden type is `Vec<T>`\nkeep searching"] |
| VisitProduceDoubleton["visit `produce_doubleton`"] |
| CompareType["`produce_doubleton` hidden type is also Vec<T>\nthis matches what we saw before ✅"] |
| Done["No more items to look at in scope\nReturn `Vec<T>`"] |
| end |
| |
| BorrowCheckProduceSingleton["`borrow_check(produce_singleton)`"] |
| TypeCheckProduceSingleton["`type_check(produce_singleton)`"] |
| |
| BorrowCheckProduceDoubleton["`borrow_check(produce_doubleton)`"] |
| TypeCheckProduceDoubleton["`type_check(produce_doubleton)`"] |
| |
| Substitute["Substitute `T => u32`,\nyielding `Vec<i32>` as the hidden type"] |
| CheckSend["Check that `Vec<i32>: Send` ✅"] |
| |
| TypeChecking -- trait code for auto traits --> TypeOfSeq |
| TypeOfSeq --> WalkModuleHir |
| WalkModuleHir --> VisitProduceSingleton |
| VisitProduceSingleton --> BorrowCheckProduceSingleton |
| BorrowCheckProduceSingleton --> TypeCheckProduceSingleton |
| TypeCheckProduceSingleton --> InterimType |
| InterimType --> VisitProduceDoubleton |
| VisitProduceDoubleton --> BorrowCheckProduceDoubleton |
| BorrowCheckProduceDoubleton --> TypeCheckProduceDoubleton |
| TypeCheckProduceDoubleton --> CompareType --> Done |
| Done --> Substitute --> CheckSend |
| </pre> |
| <h3 id="within-the-type_of-query"><a class="header" href="#within-the-type_of-query">Within the <code>type_of</code> query</a></h3> |
| <p>The <code>type_of</code> query, when applied to an opaque type O, returns the hidden type. |
| That hidden type is computed by combining the results |
| from each constraining function within the defining scope of O.</p> |
| <pre class="mermaid">flowchart TD |
| TypeOf["type_of query"] |
| TypeOf -- find_opaque_ty_constraints --> FindOpaqueTyConstraints |
| FindOpaqueTyConstraints --> Iterate |
| Iterate["Iterate over each item in defining scope"] |
| Iterate -- For each item --> TypeCheck |
| TypeCheck["Check typeck(I) to see if it constraints O"] |
| TypeCheck -- I does not\nconstrain O --> Iterate |
| TypeCheck -- I constrains O --> BorrowCheck |
| BorrowCheck["Invoke mir_borrowck(I) to get hidden type\nfor O computed by I"] |
| BorrowCheck --> PreviousType |
| PreviousType["Hidden type from I\nsame as any previous hidden type\nfound so far?"] |
| PreviousType -- Yes --> Complete |
| PreviousType -- No --> ReportError |
| ReportError["Report an error"] |
| ReportError --> Complete["Item I complete"] |
| Complete --> Iterate |
| |
| FindOpaqueTyConstraints -- All constraints found --> Done |
| Done["Done"] |
| </pre> |
| <h3 id="relating-an-opaque-type-to-another-type"><a class="header" href="#relating-an-opaque-type-to-another-type">Relating an opaque type to another type</a></h3> |
| <p>There is one central place where an opaque type gets its hidden type constrained, |
| and that is the <code>handle_opaque_type</code> function. |
| Amusingly it takes two types, so you can pass any two types, |
| but one of them should be an opaque type. |
| The order is only important for diagnostics.</p> |
| <pre class="mermaid">flowchart TD |
| subgraph typecheck["type check comparison routines"] |
| equate.rs |
| sub.rs |
| lub.rs |
| end |
| |
| typecheck --> TwoSimul |
| |
| subgraph handleopaquetype["infcx.handle_opaque_type"] |
| |
| TwoSimul["Defining two opaque types simultaneously?"] |
| |
| TwoSimul -- Yes --> ReportError["Report error"] |
| |
| TwoSimul -- No --> MayDefine -- Yes --> RegisterOpaqueType --> AlreadyHasValue |
| |
| MayDefine -- No --> ReportError |
| |
| MayDefine["In defining scope OR in query?"] |
| |
| AlreadyHasValue["Opaque type X already has\na registered value?"] |
| |
| AlreadyHasValue -- No --> Obligations["Register opaque type bounds\nas obligations for hidden type"] |
| |
| RegisterOpaqueType["Register opaque type with\nother type as value"] |
| |
| AlreadyHasValue -- Yes --> EquateOpaqueTypes["Equate new hidden type\nwith old hidden type"] |
| end |
| </pre> |
| <h3 id="interactions-with-queries"><a class="header" href="#interactions-with-queries">Interactions with queries</a></h3> |
| <p>When queries handle opaque types, |
| they cannot figure out whether they are in a defining scope, |
| so they just assume they are.</p> |
| <p>The registered hidden types are stored into the <code>QueryResponse</code> struct |
| in the <code>opaque_types</code> field (the function |
| <code>take_opaque_types_for_query_response</code> reads them out).</p> |
| <p>When the <code>QueryResponse</code> is instantiated into the surrounding infcx in |
| <code>query_response_substitution_guess</code>, |
| we convert each hidden type constraint by invoking <code>handle_opaque_type</code> (as above).</p> |
| <p>There is one bit of "weirdness". |
| The instantiated opaque types have an order |
| (if one opaque type was compared with another, |
| and we have to pick one opaque type to use as the one that gets its hidden type assigned). |
| We use the one that is considered "expected". |
| But really both of the opaque types may have defining uses. |
| When the query result is instantiated, |
| that will be re-evaluated from the context that is using the query. |
| The final context (typeck of a function, mir borrowck or wf-checks) |
| will know which opaque type can actually be instantiated |
| and then handle it correctly.</p> |
| <h3 id="within-the-mir-borrow-checker"><a class="header" href="#within-the-mir-borrow-checker">Within the MIR borrow checker</a></h3> |
| <p>The MIR borrow checker relates things via <code>nll_relate</code> and only cares about regions. |
| Any type relation will trigger the binding of hidden types, |
| so the borrow checker is doing the same thing as the type checker, |
| but ignores obviously dead code (e.g. after a panic). |
| The borrow checker is also the source of truth when it comes to hidden types, |
| as it is the only one who can properly figure out what lifetimes on the hidden type correspond |
| to which lifetimes on the opaque type declaration.</p> |
| <h2 id="backwards-compatibility-hacks"><a class="header" href="#backwards-compatibility-hacks">Backwards compatibility hacks</a></h2> |
| <p><code>impl Trait</code> in return position has various quirks that were not part |
| of any RFCs and are likely accidental stabilization. |
| To support these, |
| the <code>replace_opaque_types_with_inference_vars</code> is being used to reintroduce the previous behaviour.</p> |
| <p>There are three backwards compatibility hacks:</p> |
| <ol> |
| <li> |
| <p>All return sites share the same inference variable, |
| so some return sites may only compile if another return site uses a concrete type.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo() -> impl Debug { |
| if false { |
| return std::iter::empty().collect(); |
| } |
| vec![42] |
| } |
| <span class="boring">}</span></code></pre></pre> |
| </li> |
| <li> |
| <p>Associated type equality constraints for <code>impl Trait</code> can be used |
| as long as the hidden type satisfies the trait bounds on the associated type. |
| The opaque <code>impl Trait</code> signature does not need to satisfy them.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Duh {} |
| |
| impl Duh for i32 {} |
| |
| trait Trait { |
| type Assoc: Duh; |
| } |
| |
| // the fact that `R` is the `::Output` projection on `F` causes |
| // an intermediate inference var to be generated which is then later |
| // compared against the actually found `Assoc` type. |
| impl<R: Duh, F: FnMut() -> R> Trait for F { |
| type Assoc = R; |
| } |
| |
| // The `impl Send` here is then later compared against the inference var |
| // created, causing the inference var to be set to `impl Send` instead of |
| // the hidden type. We already have obligations registered on the inference |
| // var to make it uphold the `: Duh` bound on `Trait::Assoc`. The opaque |
| // type does not implement `Duh`, even if its hidden type does. |
| // Lazy TAIT would error out, but we inserted a hack to make it work again, |
| // keeping backwards compatibility. |
| fn foo() -> impl Trait<Assoc = impl Send> { |
| || 42 |
| } |
| <span class="boring">}</span></code></pre></pre> |
| </li> |
| <li> |
| <p>Closures cannot create hidden types for their parent function's <code>impl Trait</code>. |
| This point is mostly moot, |
| because of point 1 introducing inference vars, |
| so the closure only ever sees the inference var, but should we fix 1, this will become a problem.</p> |
| </li> |
| </ol> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="return-position-impl-trait-in-trait"><a class="header" href="#return-position-impl-trait-in-trait">Return Position Impl Trait In Trait</a></h1> |
| <p>Return-position impl trait in trait (RPITIT) is conceptually (and as of |
| <a href="https://github.com/rust-lang/rust/pull/112988">#112988</a>, literally) sugar that turns RPITs in trait methods into |
| generic associated types (GATs) without the user having to define that |
| GAT either on the trait side or impl side.</p> |
| <p>RPITIT was originally implemented in <a href="https://github.com/rust-lang/rust/pull/101224">#101224</a>, which added support for |
| async fn in trait (AFIT), since the implementation for RPITIT came for |
| free as a part of implementing AFIT which had been RFC'd previously. It |
| was then RFC'd independently in <a href="https://github.com/rust-lang/rfcs/pull/3425">RFC 3425</a>, which was recently approved |
| by T-lang.</p> |
| <h2 id="how-does-it-work-1"><a class="header" href="#how-does-it-work-1">How does it work?</a></h2> |
| <p>This doc is ordered mostly via the compilation pipeline:</p> |
| <ol> |
| <li>AST lowering (AST -> HIR)</li> |
| <li>HIR ty lowering (HIR -> rustc_middle::ty data types)</li> |
| <li>typeck</li> |
| </ol> |
| <h3 id="ast-lowering-2"><a class="header" href="#ast-lowering-2">AST lowering</a></h3> |
| <p>AST lowering for RPITITs is almost the same as lowering RPITs. We |
| still lower them as |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.OpaqueTy.html"><code>hir::ItemKind::OpaqueTy</code></a>. |
| The two differences are that:</p> |
| <p>We record <code>in_trait</code> for the opaque. This will signify that the opaque |
| is an RPITIT for HIR ty lowering, diagnostics that deal with HIR, etc.</p> |
| <p>We record <code>lifetime_mapping</code>s for the opaque type, described below.</p> |
| <h4 id="aside-opaque-lifetime-duplication"><a class="header" href="#aside-opaque-lifetime-duplication">Aside: Opaque lifetime duplication</a></h4> |
| <p><em>All opaques</em> (not just RPITITs) end up duplicating their captured |
| lifetimes into new lifetime parameters local to the opaque. The main |
| reason we do this is because RPITs need to be able to "reify"<sup class="footnote-reference" id="fr-1-1"><a href="#footnote-1">1</a></sup> any |
| captured late-bound arguments, or make them into early-bound ones. This |
| is so they can be used as generic args for the opaque, and later to |
| instantiate hidden types. Since we don't know which lifetimes are early- |
| or late-bound during AST lowering, we just do this for all lifetimes.</p> |
| <p>The main addition for RPITITs is that during lowering we track the |
| relationship between the captured lifetimes and the corresponding |
| duplicated lifetimes in an additional field, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.OpaqueTy.html#structfield.lifetime_mapping"><code>OpaqueTy::lifetime_mapping</code></a>. |
| We use this lifetime mapping later on in <code>predicates_of</code> to install |
| bounds that enforce equality between these duplicated lifetimes and |
| their source lifetimes in order to properly typecheck these GATs, which |
| will be discussed below.</p> |
| <h5 id="note-1"><a class="header" href="#note-1">Note</a></h5> |
| <p>It may be better if we were able to lower without duplicates and for |
| that I think we would need to stop distinguishing between early and late |
| bound lifetimes. So we would need a solution like <a href="https://github.com/rust-lang/rust/pull/103448">Account for |
| late-bound lifetimes in generics |
| #103448</a> and then also a |
| PR similar to <a href="https://github.com/rust-lang/rust/pull/103449">Inherit function lifetimes for impl-trait |
| #103449</a>.</p> |
| <h3 id="hir-ty-lowering"><a class="header" href="#hir-ty-lowering">HIR ty lowering</a></h3> |
| <p>The main change to HIR ty lowering is that we lower <code>hir::TyKind::OpaqueDef</code> |
| for an RPITIT to a projection instead of an opaque, using a newly |
| synthesized def-id for a new associated type in the trait. We'll |
| describe how exactly we get this def-id in the next section.</p> |
| <p>This means that any time we call <code>lower_ty</code> on the RPITIT, we end up |
| getting a projection back instead of an opaque. This projection can then |
| be normalized to the right value -- either the original opaque if we're |
| in the trait, or the inferred type of the RPITIT if we're in an impl.</p> |
| <h4 id="lowering-to-synthetic-associated-types"><a class="header" href="#lowering-to-synthetic-associated-types">Lowering to synthetic associated types</a></h4> |
| <p>Using query feeding, we synthesize new associated types on both the |
| trait side and impl side for RPITITs that show up in methods.</p> |
| <h5 id="lowering-rpitits-in-traits"><a class="header" href="#lowering-rpitits-in-traits">Lowering RPITITs in traits</a></h5> |
| <p>When <code>tcx.associated_item_def_ids(trait_def_id)</code> is called on a trait to |
| gather all of the trait's associated types, the query previously just |
| returned the def-ids of the HIR items that are children of the trait. |
| After <a href="https://github.com/rust-lang/rust/pull/112988">#112988</a>, additionally, for each method in the trait, we add the |
| def-ids returned by |
| <code>tcx.associated_types_for_impl_traits_in_associated_fn(trait_method_def_id)</code>, |
| which walks through each trait method, gathers any RPITITs that show up |
| in the signature, and then calls |
| <code>associated_type_for_impl_trait_in_trait</code> for each RPITIT, which |
| synthesizes a new associated type.</p> |
| <h5 id="lowering-rpitits-in-impls"><a class="header" href="#lowering-rpitits-in-impls">Lowering RPITITs in impls</a></h5> |
| <p>Similarly, along with the impl's HIR items, for each impl method, we |
| additionally add all of the |
| <code>associated_types_for_impl_traits_in_associated_fn</code> for the impl method. |
| This calls <code>associated_type_for_impl_trait_in_impl</code>, which will |
| synthesize an associated type definition for each RPITIT that comes from |
| the corresponding trait method.</p> |
| <h4 id="synthesizing-new-associated-types"><a class="header" href="#synthesizing-new-associated-types">Synthesizing new associated types</a></h4> |
| <p>We use query feeding |
| (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/plumbing/struct.TyCtxtAt.html#method.create_def"><code>TyCtxtAt::create_def</code></a>) |
| to synthesize a new def-id for the synthetic GATs for each RPITIT.</p> |
| <p>Locally, most of rustc's queries match on the HIR of an item to compute |
| their values. Since the RPITIT doesn't really have HIR associated with |
| it, or at least not HIR that corresponds to an associated type, we must |
| compute many queries eagerly and |
| <a href="https://github.com/rust-lang/rust/pull/104940">feed</a> them, like |
| <code>opt_def_kind</code>, <code>associated_item</code>, <code>visibility</code>, and<code>defaultness</code>.</p> |
| <p>The values for most of these queries is obvious, since the RPITIT |
| conceptually inherits most of its information from the parent function |
| (e.g. <code>visibility</code>), or because it's trivially knowable because it's an |
| associated type (<code>opt_def_kind</code>).</p> |
| <p>Some other queries are more involved, or cannot be fed, and we |
| document the interesting ones of those below:</p> |
| <h5 id="generics_of-for-the-trait"><a class="header" href="#generics_of-for-the-trait"><code>generics_of</code> for the trait</a></h5> |
| <p>The GAT for an RPITIT conceptually inherits the same generics as the |
| RPIT it comes from. However, instead of having the method as the |
| generics' parent, the trait is the parent.</p> |
| <p>Currently we get away with taking the RPIT's generics and method |
| generics and flattening them both into a new generics list, preserving |
| the def-id of each of the parameters. (This may cause issues with |
| def-ids having the wrong parents, but in the worst case this will cause |
| diagnostics issues. If this ends up being an issue, we can synthesize |
| new def-ids for generic params whose parent is the GAT.)</p> |
| <details> |
| <summary> <b>An illustrated example</b> </summary> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo { |
| fn method<'early: 'early, 'late, T>() -> impl Sized + Captures<'early, 'late>; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Would desugar to...</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo { |
| // vvvvvvvvv method's generics |
| // vvvvvvvvvvvvvvvvvvvvvvvv opaque's generics |
| type Gat<'early, T, 'early_duplicated, 'late>: Sized + Captures<'early_duplicated, 'late>; |
| |
| fn method<'early: 'early, 'late, T>() -> Self::Gat<'early, T, 'early, 'late>; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| </details> |
| <h5 id="generics_of-for-the-impl"><a class="header" href="#generics_of-for-the-impl"><code>generics_of</code> for the impl</a></h5> |
| <p>The generics for an impl's GAT are a bit more interesting. They are |
| composed of RPITIT's own generics (from the trait definition), appended |
| onto the impl's methods generics. This has the same issue as above, |
| where the generics for the GAT have parameters whose def-ids have the |
| wrong parent, but this should only cause issues in diagnostics.</p> |
| <p>We could fix this similarly if we were to synthesize new generics |
| def-ids, but this can be done later in a forwards-compatible way, |
| perhaps by a interested new contributor.</p> |
| <h5 id="opt_rpitit_info"><a class="header" href="#opt_rpitit_info"><code>opt_rpitit_info</code></a></h5> |
| <p>Some queries rely on computing information that would result in cycles |
| if we were to feed them eagerly, like <code>explicit_predicates_of</code>. |
| Therefore we defer to the <code>predicates_of</code> provider to return the right |
| value for our RPITIT's GAT. We do this by detecting early on in the |
| query if the associated type is synthetic by using |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.opt_rpitit_info"><code>opt_rpitit_info</code></a>, |
| which returns <code>Some</code> if the associated type is synthetic.</p> |
| <p>Then, during a query like <code>explicit_predicates_of</code>, we can detect if an |
| associated type is synthetic like:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ... { |
| if let Some(rpitit_info) = tcx.opt_rpitit_info(def_id) { |
| // Do something special for RPITITs... |
| return ...; |
| } |
| |
| // The regular computation which relies on access to the HIR of `def_id`. |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h5 id="explicit_predicates_of"><a class="header" href="#explicit_predicates_of"><code>explicit_predicates_of</code></a></h5> |
| <p>RPITITs begin by copying the predicates of the method that defined it, |
| both on the trait and impl side.</p> |
| <p>Additionally, we install "bidirectional outlives" predicates. |
| Specifically, we add region-outlives predicates in both directions for |
| each captured early-bound lifetime that constrains it to be equal to the |
| duplicated early-bound lifetime that results from lowering. This is best |
| illustrated in an example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo<'a> { |
| fn bar() -> impl Sized + 'a; |
| } |
| |
| // Desugars into... |
| |
| trait Foo<'a> { |
| type Gat<'a_duplicated>: Sized + 'a |
| where |
| 'a: 'a_duplicated, |
| 'a_duplicated: 'a; |
| //~^ Specifically, we should be able to assume that the |
| // duplicated `'a_duplicated` lifetime always stays in |
| // sync with the `'a` lifetime. |
| |
| fn bar() -> Self::Gat<'a>; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h5 id="assumed_wf_types"><a class="header" href="#assumed_wf_types"><code>assumed_wf_types</code></a></h5> |
| <p>The GATs in both the trait and impl inherit the <code>assumed_wf_types</code> of |
| the trait method that defines the RPITIT. This is to make sure that the |
| following code is well formed when lowered.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo { |
| fn iter<'a, T>(x: &'a [T]) -> impl Iterator<Item = &'a T>; |
| } |
| |
| // which is lowered to... |
| |
| trait FooDesugared { |
| type Iter<'a, T>: Iterator<Item = &'a T>; |
| //~^ assumed wf: `&'a [T]` |
| // Without assumed wf types, the GAT would not be well-formed on its own. |
| |
| fn iter<'a, T>(x: &'a [T]) -> Self::Iter<'a, T>; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Because <code>assumed_wf_types</code> is only defined for local def ids, in order |
| to properly implement <code>assumed_wf_types</code> for impls of foreign traits |
| with RPITs, we need to encode the assumed wf types of RPITITs in an |
| extern query |
| <a href="https://github.com/rust-lang/rust/blob/a17c7968b727d8413801961fc4e89869b6ab00d3/compiler/rustc_ty_utils/src/implied_bounds.rs#L14"><code>assumed_wf_types_for_rpitit</code></a>.</p> |
| <h3 id="typechecking"><a class="header" href="#typechecking">Typechecking</a></h3> |
| <h4 id="the-rpitit-inference-algorithm"><a class="header" href="#the-rpitit-inference-algorithm">The RPITIT inference algorithm</a></h4> |
| <p>The RPITIT inference algorithm is implemented in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.collect_return_position_impl_trait_in_trait_tys.html"><code>collect_return_position_impl_trait_in_trait_tys</code></a>.</p> |
| <p><strong>High-level:</strong> Given a impl method and a trait method, we take the |
| trait method and instantiate each RPITIT in the signature with an infer |
| var. We then equate this trait method signature with the impl method |
| signature, and process all obligations that fall out in order to infer |
| the type of all of the RPITITs in the method.</p> |
| <p>The method is also responsible for making sure that the hidden types for |
| each RPITIT actually satisfy the bounds of the <code>impl Trait</code>, i.e. that |
| if we infer <code>impl Trait = Foo</code>, that <code>Foo: Trait</code> holds.</p> |
| <details> |
| <summary><b>An example...</b></summary> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(return_position_impl_trait_in_trait)] |
| |
| <span class="boring">fn main() { |
| </span>use std::ops::Deref; |
| |
| trait Foo { |
| fn bar() -> impl Deref<Target = impl Sized>; |
| // ^- RPITIT ?0 ^- RPITIT ?1 |
| } |
| |
| impl Foo for () { |
| fn bar() -> Box<String> { Box::new(String::new()) } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>We end up with the trait signature that looks like <code>fn() -> ?0</code>, and |
| nested obligations <code>?0: Deref<Target = ?1></code>, <code>?1: Sized</code>. The impl |
| signature is <code>fn() -> Box<String></code>.</p> |
| <p>Equating these signatures gives us <code>?0 = Box<String></code>, which then after |
| processing the obligation <code>Box<String>: Deref<Target = ?1></code> gives us <code>?1 = String</code>, and the other obligation <code>String: Sized</code> evaluates to true.</p> |
| <p>By the end of the algorithm, we end up with a mapping between associated |
| type def-ids to concrete types inferred from the signature. We can then |
| use this mapping to implement <code>type_of</code> for the synthetic associated |
| types in the impl, since this mapping describes the type that should |
| come after the <code>=</code> in <code>type Assoc = ...</code> for each RPITIT.</p> |
| </details> |
| <h5 id="implied-bounds-in-rpitit-hidden-type-inference"><a class="header" href="#implied-bounds-in-rpitit-hidden-type-inference">Implied bounds in RPITIT hidden type inference</a></h5> |
| <p>Since <code>collect_return_position_impl_trait_in_trait_tys</code> does fulfillment and |
| region resolution, we must provide it <code>assumed_wf_types</code> so that we can prove |
| region obligations with the same expected implied bounds as |
| <code>compare_method_predicate_entailment</code> does.</p> |
| <p>Since the return type of a method is understood to be one of the assumed WF |
| types, and we eagerly fold the return type with inference variables to do |
| opaque type inference, after opaque type inference, the return type will |
| resolve to contain the hidden types of the RPITITs. this would mean that the |
| hidden types of the RPITITs would be assumed to be well-formed without having |
| independently proven that they are. This resulted in a |
| <a href="https://github.com/rust-lang/rust/pull/116072">subtle unsoundness bug</a>. In |
| order to prevent this cyclic reasoning, we instead replace the hidden types of |
| the RPITITs in the return type of the method with <em>placeholders</em>, which lead |
| to no implied well-formedness bounds.</p> |
| <h4 id="default-trait-body"><a class="header" href="#default-trait-body">Default trait body</a></h4> |
| <p>Type-checking a default trait body, like:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo { |
| fn bar() -> impl Sized { |
| 1i32 |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>requires one interesting hack. We need to install a projection predicate |
| into the param-env of <code>Foo::bar</code> allowing us to assume that the RPITIT's |
| GAT normalizes to the RPITIT's opaque type. This relies on the |
| observation that a trait method and RPITIT's GAT will always be "in |
| sync". That is, one will only ever be overridden if the other one is as |
| well.</p> |
| <p>Compare this to a similar desugaring of the code above, which would fail |
| because we cannot rely on this same assumption:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(impl_trait_in_assoc_type)] |
| #![feature(associated_type_defaults)] |
| |
| <span class="boring">fn main() { |
| </span>trait Foo { |
| type RPITIT = impl Sized; |
| |
| fn bar() -> Self::RPITIT { |
| 01i32 |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Failing because a down-stream impl could theoretically provide an |
| implementation for <code>RPITIT</code> without providing an implementation of |
| <code>bar</code>:</p> |
| <pre><code class="language-text">error[E0308]: mismatched types |
| --> src/lib.rs:8:9 |
| | |
| 5 | type RPITIT = impl Sized; |
| | ------------------------- associated type defaults can't be assumed inside the trait defining them |
| 6 | |
| 7 | fn bar() -> Self::RPITIT { |
| | ------------ expected `<Self as Foo>::RPITIT` because of return type |
| 8 | 01i32 |
| | ^^^^^ expected associated type, found `i32` |
| | |
| = note: expected associated type `<Self as Foo>::RPITIT` |
| found type `i32` |
| </code></pre> |
| <h4 id="well-formedness-checking"><a class="header" href="#well-formedness-checking">Well-formedness checking</a></h4> |
| <p>We check well-formedness of RPITITs just like regular associated types.</p> |
| <p>Since we added lifetime bounds in <code>predicates_of</code> that link the |
| duplicated early-bound lifetimes to their original lifetimes, and we |
| implemented <code>assumed_wf_types</code> which inherits the WF types of the method |
| from which the RPITIT originates (<a href="https://github.com/rust-lang/rust/pull/113704">#113704</a>), we have no issues |
| WF-checking the GAT as if it were a regular GAT.</p> |
| <h3 id="whats-broken-whats-weird-etc"><a class="header" href="#whats-broken-whats-weird-etc">What's broken, what's weird, etc.</a></h3> |
| <h5 id="specialization-is-super-busted"><a class="header" href="#specialization-is-super-busted">Specialization is super busted</a></h5> |
| <p>The "default trait methods" described above does not interact well with |
| specialization, because we only install those projection bounds in trait |
| default methods, and not in impl methods. Given that specialization is |
| already pretty busted, I won't go into detail, but it's currently a bug |
| tracked in: |
| * <code>tests/ui/impl-trait/in-trait/specialization-broken.rs</code></p> |
| <h5 id="projections-dont-have-variances"><a class="header" href="#projections-dont-have-variances">Projections don't have variances</a></h5> |
| <p>This code fails because projections don't have variances:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(return_position_impl_trait_in_trait)] |
| |
| <span class="boring">fn main() { |
| </span>trait Foo { |
| // Note that the RPITIT below does *not* capture `'lt`. |
| fn bar<'lt: 'lt>() -> impl Eq; |
| } |
| |
| fn test<'a, 'b, T: Foo>() -> bool { |
| <T as Foo>::bar::<'a>() == <T as Foo>::bar::<'b>() |
| //~^ ERROR |
| // (requires that `'a == 'b`) |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This is because we can't relate <code><T as Foo>::Rpitit<'a></code> and <code><T as Foo>::Rpitit<'b></code>, even if they don't capture their lifetime. If we were |
| using regular opaque types, this would work, because they would be |
| bivariant in that lifetime parameter:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![feature(return_position_impl_trait_in_trait)] |
| |
| <span class="boring">fn main() { |
| </span>fn bar<'lt: 'lt>() -> impl Eq { |
| () |
| } |
| |
| fn test<'a, 'b>() -> bool { |
| bar::<'a>() == bar::<'b>() |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>This is probably okay though, since RPITITs will likely have their |
| captures behavior changed to capture all in-scope lifetimes anyways. |
| This could also be relaxed later in a forwards-compatible way if we were |
| to consider variances of RPITITs when relating projections.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-1"> |
| <p>This is compiler-errors terminology, I'm not claiming it's accurate :^) <a href="#fr-1-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="opaque-types-region-inference-restrictions"><a class="header" href="#opaque-types-region-inference-restrictions">Opaque types region inference restrictions</a></h1> |
| <p>In this chapter we discuss the various restrictions we impose on the generic arguments of |
| opaque types when defining their hidden types |
| <code>Opaque<'a, 'b, .., A, B, ..> := SomeHiddenType</code>.</p> |
| <p>These restrictions are implemented in borrow checking (<a href="https://github.com/rust-lang/rust/blob/435b5255148617128f0a9b17bacd3cc10e032b23/compiler/rustc_borrowck/src/region_infer/opaque_types.rs">Source</a>) |
| as it is the final step opaque types inference.</p> |
| <h2 id="background-type-and-const-generic-arguments"><a class="header" href="#background-type-and-const-generic-arguments">Background: type and const generic arguments</a></h2> |
| <p>For type arguments, two restrictions are necessary: each type argument must be |
| (1) a type parameter and |
| (2) is unique among the generic arguments. |
| The same is applied to const arguments.</p> |
| <p>Example of case (1):</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<X> = impl Sized; |
| |
| // `T` is a type parameter. |
| // Opaque<T> := (); |
| fn good<T>() -> Opaque<T> {} |
| |
| // `()` is not a type parameter. |
| // Opaque<()> := (); |
| fn bad() -> Opaque<()> {} //~ ERROR |
| <span class="boring">}</span></code></pre></pre> |
| <p>Example of case (2):</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<X, Y> = impl Sized; |
| |
| // `T` and `U` are unique in the generic args. |
| // Opaque<T, U> := T; |
| fn good<T, U>(t: T, _u: U) -> Opaque<T, U> { t } |
| |
| // `T` appears twice in the generic args. |
| // Opaque<T, T> := T; |
| fn bad<T>(t: T) -> Opaque<T, T> { t } //~ ERROR |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Motivation:</strong> In the first case <code>Opaque<()> := ()</code>, the hidden type is ambiguous because |
| it is compatible with two different interpretaions: <code>Opaque<X> := X</code> and <code>Opaque<X> := ()</code>. |
| Similarly for the second case <code>Opaque<T, T> := T</code>, it is ambiguous whether it should be |
| interpreted as <code>Opaque<X, Y> := X</code> or as <code>Opaque<X, Y> := Y</code>. |
| Because of this ambiguity, both cases are rejected as invalid defining uses.</p> |
| <h2 id="uniqueness-restriction"><a class="header" href="#uniqueness-restriction">Uniqueness restriction</a></h2> |
| <p>Each lifetime argument must be unique in the arguments list and must not be <code>'static</code>. |
| This is in order to avoid an ambiguity with hidden type inference similar to the case of |
| type parameters. |
| For example, the invalid defining use below <code>Opaque<'static> := Inv<'static></code> is compatible with |
| both <code>Opaque<'x> := Inv<'static></code> and <code>Opaque<'x> := Inv<'x></code>.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<'x> = impl Sized + 'x; |
| type Inv<'a> = Option<*mut &'a ()>; |
| |
| fn good<'a>() -> Opaque<'a> { Inv::<'static>::None } |
| |
| fn bad() -> Opaque<'static> { Inv::<'static>::None } |
| //~^ ERROR |
| <span class="boring">}</span></code></pre></pre> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<'x, 'y> = impl Trait<'x, 'y>; |
| |
| fn good<'a, 'b>() -> Opaque<'a, 'b> {} |
| |
| fn bad<'a>() -> Opaque<'a, 'a> {} |
| //~^ ERROR |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Semantic lifetime equality:</strong> |
| One complexity with lifetimes compared to type parameters is that |
| two lifetimes that are syntactically different may be semantically equal. |
| Therefore, we need to be cautious when verifying that the lifetimes are unique.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// This is also invalid because `'a` is *semantically* equal to `'static`. |
| fn still_bad_1<'a: 'static>() -> Opaque<'a> {} |
| //~^ Should error! |
| |
| // This is also invalid because `'a` and `'b` are *semantically* equal. |
| fn still_bad_2<'a: 'b, 'b: 'a>() -> Opaque<'a, 'b> {} |
| //~^ Should error! |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="an-exception-to-uniqueness-rule"><a class="header" href="#an-exception-to-uniqueness-rule">An exception to uniqueness rule</a></h2> |
| <p>An exception to the uniqueness rule above is when the bounds at the opaque type's definition require |
| a lifetime parameter to be equal to another one or to the <code>'static</code> lifetime.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// The definition requires `'x` to be equal to `'static`. |
| type Opaque<'x: 'static> = impl Sized + 'x; |
| |
| fn good() -> Opaque<'static> {} |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Motivation:</strong> an attempt to implement the uniqueness restriction for RPITs resulted in a |
| <a href="https://github.com/rust-lang/rust/pull/112842#issuecomment-1610057887">breakage found by crater</a>. |
| This can be mitigated by this exception to the rule. |
| An example of the code that would otherwise break:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct Type<'a>(&'a ()); |
| impl<'a> Type<'a> { |
| // `'b == 'a` |
| fn do_stuff<'b: 'a>(&'b self) -> impl Trait<'a, 'b> {} |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Why this is correct:</strong> for such a defining use like <code>Opaque<'a, 'a> := &'a str</code>, |
| it can be interpreted in either way—either as <code>Opaque<'x, 'y> := &'x str</code> or as |
| <code>Opaque<'x, 'y> := &'y str</code> and it wouldn't matter because every use of <code>Opaque</code> |
| will guarantee that both parameters are equal as per the well-formedness rules.</p> |
| <h2 id="universal-lifetimes-restriction"><a class="header" href="#universal-lifetimes-restriction">Universal lifetimes restriction</a></h2> |
| <p>Only universally quantified lifetimes are allowed in the opaque type arguments. |
| This includes lifetime parameters and placeholders.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<'x> = impl Sized + 'x; |
| |
| fn test<'a>() -> Opaque<'a> { |
| // `Opaque<'empty> := ()` |
| let _: Opaque<'_> = (); |
| //~^ ERROR |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Motivation:</strong> |
| This makes the lifetime and type arguments behave consistently but this is only as a bonus. |
| The real reason behind this restriction is purely technical, as the <a href="borrow_check/./region_inference/member_constraints.html">member constraints</a> algorithm |
| faces a fundamental limitation: |
| When encountering an opaque type definition <code>Opaque<'?1> := &'?2 u8</code>, |
| a member constraint <code>'?2 member-of ['static, '?1]</code> is registered. |
| In order for the algorithm to pick the right choice, the <em>complete</em> set of "outlives" relationships |
| between the choice regions <code>['static, '?1]</code> must already be known <em>before</em> doing the region |
| inference. This can be satisfied only if each choice region is either:</p> |
| <ol> |
| <li>a universal region, i.e. <code>RegionKind::Re{EarlyParam,LateParam,Placeholder,Static}</code>, |
| because the relations between universal regions are completely known, prior to region inference, |
| from the explicit and implied bounds.</li> |
| <li>or an existential region that is "strictly equal" to a universal region. |
| Strict lifetime equality is defined below and is required here because it is the only type of |
| equality that can be evaluated prior to full region inference.</li> |
| </ol> |
| <p><strong>Strict lifetime equality:</strong> |
| We say that two lifetimes are strictly equal if there are bidirectional outlives constraints |
| between them. In NLL terms, this means the lifetimes are part of the same <a href="https://en.wikipedia.org/wiki/Strongly_connected_component">SCC</a>. |
| Importantly this type of equality can be evaluated prior to full region inference |
| (but of course after constraint collection). |
| The other type of equality is when region inference ends up giving two lifetimes variables |
| the same value even if they are not strictly equal. |
| See <a href="https://github.com/rust-lang/rust/issues/113971">#113971</a> for how we used to conflate the difference.</p> |
| <p><strong>interaction with "once modulo regions" restriction</strong> |
| In the example above, note the opaque type in the signature is <code>Opaque<'a></code> and the one in the |
| invalid defining use is <code>Opaque<'empty></code>. |
| In the proposed MiniTAIT plan, namely the <a href="https://github.com/rust-lang/rust/pull/116935">"once modulo regions"</a> rule, |
| we already disallow this. |
| Although it might appear that "universal lifetimes" restriction becomes redundant as it logically |
| follows from "MiniTAIT" restrictions, the subsequent related discussion on lifetime equality and |
| closures remains relevant.</p> |
| <h2 id="closure-restrictions"><a class="header" href="#closure-restrictions">Closure restrictions</a></h2> |
| <p>When the opaque type is defined in a closure/coroutine/inline-const body, universal lifetimes that |
| are "external" to the closure are not allowed in the opaque type arguments. |
| External regions are defined in <a href="https://github.com/rust-lang/rust/blob/caf730043232affb6b10d1393895998cb4968520/compiler/rustc_borrowck/src/universal_regions.rs#L201."><code>RegionClassification::External</code></a></p> |
| <p>Example: (This one happens to compile in the current nightly but more practical examples are |
| already rejected with confusing errors.)</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<'x> = impl Sized + 'x; |
| |
| fn test<'a>() -> Opaque<'a> { |
| let _ = || { |
| // `'a` is external to the closure |
| let _: Opaque<'a> = (); |
| //~^ Should be an error! |
| }; |
| () |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p><strong>Motivation:</strong> |
| In closure bodies, external lifetimes, although being categorized as "universal" lifetimes, |
| behave more like existential lifetimes in that the relations between them are not known ahead of |
| time, instead their values are inferred just like existential lifetimes and the requirements are |
| propagated back to the parent fn. This breaks the member constraints algorithm as described above:</p> |
| <blockquote> |
| <p>In order for the algorithm to pick the right choice, the complete set of “outlives” relationships |
| between the choice regions <code>['static, '?1]</code> must already be known before doing the region inference</p> |
| </blockquote> |
| <p>Here is an example that details how :</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<'x, 'y> = impl Sized; |
| |
| // |
| fn test<'a, 'b>(s: &'a str) -> impl FnOnce() -> Opaque<'a, 'b> { |
| move || { s } |
| //~^ ERROR hidden type for `Opaque<'_, '_>` captures lifetime that does not appear in bounds |
| } |
| |
| // The above closure body is desugared into something like: |
| fn test::{closure#0}(_upvar: &'?8 str) -> Opaque<'?6, '?7> { |
| return _upvar |
| } |
| |
| // where `['?8, '?6, ?7]` are universal lifetimes *external* to the closure. |
| // There are no known relations between them *inside* the closure. |
| // But in the parent fn it is known that `'?6: '?8`. |
| // |
| // When encountering an opaque definition `Opaque<'?6, '?7> := &'8 str`, |
| // The member constraints algorithm does not know enough to safely make `?8 = '?6`. |
| // For this reason, it errors with a sensible message: |
| // "hidden type captures lifetime that does not appear in bounds". |
| <span class="boring">}</span></code></pre></pre> |
| <p>Without these restrictions, error messages are confusing and, more importantly, there is a risk that |
| we accept code that would likely break in the future because member constraints are super broken |
| in closures.</p> |
| <p><strong>Output types:</strong> |
| I believe the most common scenario where this causes issues in real-world code is with |
| closure/async-block output types. It is worth noting that there is a discrepancy between closures |
| and async blocks that further demonstrates this issue and is attributed to the |
| <a href="https://github.com/rust-lang/rust/blob/9cf18e98f82d85fa41141391d54485b8747da46f/compiler/rustc_hir_typeck/src/closure.rs#L743">hack of <code>replace_opaque_types_with_inference_vars</code></a>, |
| which is applied to futures only.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>type Opaque<'x> = impl Sized + 'x; |
| fn test<'a>() -> impl FnOnce() -> Opaque<'a> { |
| // Output type of the closure is Opaque<'a> |
| // -> hidden type definition happens *inside* the closure |
| // -> rejected. |
| move || {} |
| //~^ ERROR expected generic lifetime parameter, found `'_` |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>use std::future::Future; |
| type Opaque<'x> = impl Sized + 'x; |
| fn test<'a>() -> impl Future<Output = Opaque<'a>> { |
| // Output type of the async block is unit `()` |
| // -> hidden type definition happens in the parent fn |
| // -> accepted. |
| async move {} |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="effects-and-const-condition-checking"><a class="header" href="#effects-and-const-condition-checking">Effects and const condition checking</a></h1> |
| <h2 id="the-hosteffect-predicate"><a class="header" href="#the-hosteffect-predicate">The <code>HostEffect</code> predicate</a></h2> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/predicate/struct.HostEffectPredicate.html"><code>HostEffectPredicate</code></a>s are a kind of predicate from <code>~const Tr</code> or <code>const Tr</code> |
| bounds. It has a trait reference, and a <code>constness</code> which could be <code>Maybe</code> or |
| <code>Const</code> depending on the bound. Because <code>~const Tr</code>, or rather <code>Maybe</code> bounds |
| apply differently based on whichever contexts they are in, they have different |
| behavior than normal bounds. Where normal trait bounds on a function such as |
| <code>T: Tr</code> are collected within the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.predicates_of"><code>predicates_of</code></a> query to be proven when a |
| function is called and to be assumed within the function, bounds such as |
| <code>T: ~const Tr</code> will behave as a normal trait bound and add <code>T: Tr</code> to the result |
| from <code>predicates_of</code>, but also adds a <code>HostEffectPredicate</code> to the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.const_conditions"><code>const_conditions</code></a> query.</p> |
| <p>On the other hand, <code>T: const Tr</code> bounds do not change meaning across contexts, |
| therefore they will result in <code>HostEffect(T: Tr, const)</code> being added to |
| <code>predicates_of</code>, and not <code>const_conditions</code>.</p> |
| <h2 id="the-const_conditions-query"><a class="header" href="#the-const_conditions-query">The <code>const_conditions</code> query</a></h2> |
| <p><code>predicates_of</code> represents a set of predicates that need to be proven to use an |
| item. For example, to use <code>foo</code> in the example below:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<T>() where T: Default {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>We must be able to prove that <code>T</code> implements <code>Default</code>. In a similar vein, |
| <code>const_conditions</code> represents a set of predicates that need to be proven to use |
| an item <em>in const contexts</em>. If we adjust the example above to use <code>const</code> trait |
| bounds:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>const fn foo<T>() where T: ~const Default {} |
| <span class="boring">}</span></code></pre></pre> |
| <p>Then <code>foo</code> would get a <code>HostEffect(T: Default, maybe)</code> in the <code>const_conditions</code> |
| query, suggesting that in order to call <code>foo</code> from const contexts, one must |
| prove that <code>T</code> has a const implementation of <code>Default</code>.</p> |
| <h2 id="enforcement-of-const_conditions"><a class="header" href="#enforcement-of-const_conditions">Enforcement of <code>const_conditions</code></a></h2> |
| <p><code>const_conditions</code> are currently checked in various places.</p> |
| <p>Every call in HIR from a const context (which includes <code>const fn</code> and <code>const</code> |
| items) will check that <code>const_conditions</code> of the function we are calling hold. |
| This is done in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#method.enforce_context_effects"><code>FnCtxt::enforce_context_effects</code></a>. Note that we don't check |
| if the function is only referred to but not called, as the following code needs |
| to compile:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>const fn hi<T: ~const Default>() -> T { |
| T::default() |
| } |
| const X: fn() -> u32 = hi::<u32>; |
| <span class="boring">}</span></code></pre></pre> |
| <p>For a trait <code>impl</code> to be well-formed, we must be able to prove the |
| <code>const_conditions</code> of the trait from the <code>impl</code>'s environment. This is checked |
| in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/wfcheck/fn.check_impl.html"><code>wfcheck::check_impl</code></a>.</p> |
| <p>Here's an example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>const trait Bar {} |
| const trait Foo: ~const Bar {} |
| // `const_conditions` contains `HostEffect(Self: Bar, maybe)` |
| |
| impl const Bar for () {} |
| impl const Foo for () {} |
| // ^ here we check `const_conditions` for the impl to be well-formed |
| <span class="boring">}</span></code></pre></pre> |
| <p>Methods of trait impls must not have stricter bounds than the method of the |
| trait that they are implementing. To check that the methods are compatible, a |
| hybrid environment is constructed with the predicates of the <code>impl</code> plus the |
| predicates of the trait method, and we attempt to prove the predicates of the |
| impl method. We do the same for <code>const_conditions</code>:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>const trait Foo { |
| fn hi<T: ~const Default>(); |
| } |
| |
| impl<T: ~const Clone> Foo for Vec<T> { |
| fn hi<T: ~const PartialEq>(); |
| // ^ we can't prove `T: ~const PartialEq` given `T: ~const Clone` and |
| // `T: ~const Default`, therefore we know that the method on the impl |
| // is stricter than the method on the trait. |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>These checks are done in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html"><code>compare_method_predicate_entailment</code></a>. A similar |
| function that does the same check for associated types is called |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_type_predicate_entailment.html"><code>compare_type_predicate_entailment</code></a>. Both of these need to consider |
| <code>const_conditions</code> when in const contexts.</p> |
| <p>In MIR, as part of const checking, <code>const_conditions</code> of items that are called |
| are revalidated again in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/check_consts/check/struct.Checker.html#method.revalidate_conditional_constness"><code>Checker::revalidate_conditional_constness</code></a>.</p> |
| <h2 id="explicit_implied_const_bounds-on-associated-types-and-traits"><a class="header" href="#explicit_implied_const_bounds-on-associated-types-and-traits"><code>explicit_implied_const_bounds</code> on associated types and traits</a></h2> |
| <p>Bounds on associated types, opaque types, and supertraits such as</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>trait Foo: ~const PartialEq { |
| type X: ~const PartialEq; |
| } |
| |
| fn foo() -> impl ~const PartialEq { |
| // ^ unimplemented syntax |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Have their bounds represented differently. Unlike <code>const_conditions</code> which need |
| to be proved for callers, and can be assumed inside the definition (e.g. trait |
| bounds on functions), these bounds need to be proved at definition (at the impl, |
| or when returning the opaque) but can be assumed for callers. The non-const |
| equivalent of these bounds are called <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.explicit_item_bounds"><code>explicit_item_bounds</code></a>.</p> |
| <p>These bounds are checked in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.check_type_bounds.html"><code>compare_impl_item::check_type_bounds</code></a> for HIR |
| typeck, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/effects/fn.evaluate_host_effect_from_item_bounds.html"><code>evaluate_host_effect_from_item_bounds</code></a> in the old solver and |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/assembly/trait.GoalKind.html#tymethod.consider_additional_alias_assumptions"><code>consider_additional_alias_assumptions</code></a> in the new solver.</p> |
| <h2 id="proving-hosteffectpredicates"><a class="header" href="#proving-hosteffectpredicates">Proving <code>HostEffectPredicate</code>s</a></h2> |
| <p><code>HostEffectPredicate</code>s are implemented both in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_trait_selection/traits/effects.rs.html">old solver</a> and the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_next_trait_solver/solve/effect_goals.rs.html">new |
| trait solver</a>. In general, we can prove a <code>HostEffect</code> predicate when either of |
| these conditions are met:</p> |
| <ul> |
| <li>The predicate can be assumed from caller bounds;</li> |
| <li>The type has a <code>const</code> <code>impl</code> for the trait, <em>and</em> that const conditions on |
| the impl holds, <em>and</em> that the <code>explicit_implied_const_bounds</code> on the trait |
| holds; or</li> |
| <li>The type has a built-in implementation for the trait in const contexts. For |
| example, <code>Fn</code> may be implemented by function items if their const conditions |
| are satisfied, or <code>Destruct</code> is implemented in const contexts if the type can |
| be dropped at compile time.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="pattern-and-exhaustiveness-checking"><a class="header" href="#pattern-and-exhaustiveness-checking">Pattern and exhaustiveness checking</a></h1> |
| <p>In Rust, pattern matching and bindings have a few very helpful properties. The |
| compiler will check that bindings are irrefutable when made and that match arms |
| are exhaustive.</p> |
| <h2 id="pattern-usefulness"><a class="header" href="#pattern-usefulness">Pattern usefulness</a></h2> |
| <p>The central question that usefulness checking answers is: |
| "in this match expression, is that branch redundant?". |
| More precisely, it boils down to computing whether, |
| given a list of patterns we have already seen, |
| a given new pattern might match any new value.</p> |
| <p>For example, in the following match expression, |
| we ask in turn whether each pattern might match something |
| that wasn't matched by the patterns above it. |
| Here we see the 4th pattern is redundant with the 1st; |
| that branch will get an "unreachable" warning. |
| The 3rd pattern may or may not be useful, |
| depending on whether <code>Foo</code> has other variants than <code>Bar</code>. |
| Finally, we can ask whether the whole match is exhaustive |
| by asking whether the wildcard pattern (<code>_</code>) |
| is useful relative to the list of all the patterns in that match. |
| Here we can see that <code>_</code> is useful (it would catch <code>(false, None)</code>); |
| this expression would therefore get a "non-exhaustive match" error.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// x: (bool, Option<Foo>) |
| match x { |
| (true, _) => {} // 1 |
| (false, Some(Foo::Bar)) => {} // 2 |
| (false, Some(_)) => {} // 3 |
| (true, None) => {} // 4 |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Thus usefulness is used for two purposes: |
| detecting unreachable code (which is useful to the user), |
| and ensuring that matches are exhaustive (which is important for soundness, |
| because a match expression can return a value).</p> |
| <h2 id="where-it-happens"><a class="header" href="#where-it-happens">Where it happens</a></h2> |
| <p>This check is done anywhere you can write a pattern: <code>match</code> expressions, <code>if let</code>, <code>let else</code>, |
| plain <code>let</code>, and function arguments.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// `match` |
| // Usefulness can detect unreachable branches and forbid non-exhaustive matches. |
| match foo() { |
| Ok(x) => x, |
| Err(_) => panic!(), |
| } |
| |
| // `if let` |
| // Usefulness can detect unreachable branches. |
| if let Some(x) = foo() { |
| // ... |
| } |
| |
| // `while let` |
| // Usefulness can detect infinite loops and dead loops. |
| while let Some(x) = it.next() { |
| // ... |
| } |
| |
| // Destructuring `let` |
| // Usefulness can forbid non-exhaustive patterns. |
| let Foo::Bar(x, y) = foo(); |
| |
| // Destructuring function arguments |
| // Usefulness can forbid non-exhaustive patterns. |
| fn foo(Foo { x, y }: Foo) { |
| // ... |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="the-algorithm-1"><a class="header" href="#the-algorithm-1">The algorithm</a></h2> |
| <p>Exhaustiveness checking is run before MIR building in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/pattern/check_match/index.html"><code>check_match</code></a>. |
| It is implemented in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_pattern_analysis/index.html"><code>rustc_pattern_analysis</code></a> crate, |
| with the core of the algorithm in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_pattern_analysis/usefulness/index.html"><code>usefulness</code></a> module. |
| That file contains a detailed description of the algorithm.</p> |
| <h2 id="important-concepts"><a class="header" href="#important-concepts">Important concepts</a></h2> |
| <h3 id="constructors-and-fields"><a class="header" href="#constructors-and-fields">Constructors and fields</a></h3> |
| <p>In the value <code>Pair(Some(0), true)</code>, <code>Pair</code> is called the constructor of the value, and <code>Some(0)</code> and |
| <code>true</code> are its fields. Every matchable value can be decomposed in this way. Examples of |
| constructors are: <code>Some</code>, <code>None</code>, <code>(,)</code> (the 2-tuple constructor), <code>Foo {..}</code> (the constructor for |
| a struct <code>Foo</code>), and <code>2</code> (the constructor for the number <code>2</code>).</p> |
| <p>Each constructor takes a fixed number of fields; this is called its arity. <code>Pair</code> and <code>(,)</code> have |
| arity 2, <code>Some</code> has arity 1, <code>None</code> and <code>42</code> have arity 0. Each type has a known set of |
| constructors. Some types have many constructors (like <code>u64</code>) or even an infinitely many (like <code>&str</code> |
| and <code>&[T]</code>).</p> |
| <p>Patterns are similar: <code>Pair(Some(_), _)</code> has constructor <code>Pair</code> and two fields. The difference is |
| that we get some extra pattern-only constructors, namely: the wildcard <code>_</code>, variable bindings, |
| integer ranges like <code>0..=10</code>, and variable-length slices like <code>[_, .., _]</code>. We treat or-patterns |
| separately.</p> |
| <p>Now to check if a value <code>v</code> matches a pattern <code>p</code>, we check if <code>v</code>'s constructor matches <code>p</code>'s |
| constructor, then recursively compare their fields if necessary. A few representative examples:</p> |
| <ul> |
| <li><code>matches!(v, _) := true</code></li> |
| <li><code>matches!((v0, v1), (p0, p1)) := matches!(v0, p0) && matches!(v1, p1)</code></li> |
| <li><code>matches!(Foo { a: v0, b: v1 }, Foo { a: p0, b: p1 }) := matches!(v0, p0) && matches!(v1, p1)</code></li> |
| <li><code>matches!(Ok(v0), Ok(p0)) := matches!(v0, p0)</code></li> |
| <li><code>matches!(Ok(v0), Err(p0)) := false</code> (incompatible variants)</li> |
| <li><code>matches!(v, 1..=100) := matches!(v, 1) || ... || matches!(v, 100)</code></li> |
| <li><code>matches!([v0], [p0, .., p1]) := false</code> (incompatible lengths)</li> |
| <li><code>matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)</code></li> |
| </ul> |
| <p>This concept is absolutely central to pattern analysis. The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_pattern_analysis/constructor/index.html"><code>constructor</code></a> module provides |
| functions to extract, list and manipulate constructors. This is a useful enough concept that |
| variations of it can be found in other places of the compiler, like in the MIR-lowering of a match |
| expression and in some clippy lints.</p> |
| <h3 id="constructor-grouping-and-splitting"><a class="header" href="#constructor-grouping-and-splitting">Constructor grouping and splitting</a></h3> |
| <p>The pattern-only constructors (<code>_</code>, ranges and variable-length slices) each stand for a set of |
| normal constructors, e.g. <code>_: Option<T></code> stands for the set {<code>None</code>, <code>Some</code>} and <code>[_, .., _]</code> stands |
| for the infinite set {<code>[,]</code>, <code>[,,]</code>, <code>[,,,]</code>, ...} of the slice constructors of arity >= 2.</p> |
| <p>In order to manage these constructors, we keep them as grouped as possible. For example:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>match (0, false) { |
| (0 ..=100, true) => {} |
| (50..=150, false) => {} |
| (0 ..=200, _) => {} |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example, all of <code>0</code>, <code>1</code>, .., <code>49</code> match the same arms, and thus can be treated as a group. |
| In fact, in this match, the only ranges we need to consider are: <code>0..50</code>, <code>50..=100</code>, |
| <code>101..=150</code>,<code>151..=200</code> and <code>201..</code>. Similarly:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>enum Direction { North, South, East, West } |
| <span class="boring">let wind = (Direction::North, 0u8); |
| </span>match wind { |
| (Direction::North, 50..) => {} |
| (_, _) => {} |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>Here we can treat all the non-<code>North</code> constructors as a group, giving us only two cases to handle: |
| <code>North</code>, and everything else.</p> |
| <p>This is called "constructor splitting" and is crucial to having exhaustiveness run in reasonable |
| time.</p> |
| <h3 id="usefulness-vs-reachability-in-the-presence-of-empty-types"><a class="header" href="#usefulness-vs-reachability-in-the-presence-of-empty-types">Usefulness vs reachability in the presence of empty types</a></h3> |
| <p>This is likely the subtlest aspect of exhaustiveness. To be fully precise, a match doesn't operate |
| on a value, it operates on a place. In certain unsafe circumstances, it is possible for a place to |
| not contain valid data for its type. This has subtle consequences for empty types. Take the |
| following:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>enum Void {} |
| let x: u8 = 0; |
| let ptr: *const Void = &x as *const u8 as *const Void; |
| unsafe { |
| match *ptr { |
| _ => println!("Reachable!"), |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>In this example, <code>ptr</code> is a valid pointer pointing to a place with invalid data. The <code>_</code> pattern |
| does not look at the contents of the place <code>*ptr</code>, so this code is ok and the arm is taken. In other |
| words, despite the place we are inspecting being of type <code>Void</code>, there is a reachable arm. If the |
| arm had a binding however:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span><span class="boring">#[derive(Copy, Clone)] |
| </span><span class="boring">enum Void {} |
| </span><span class="boring">let x: u8 = 0; |
| </span><span class="boring">let ptr: *const Void = &x as *const u8 as *const Void; |
| </span><span class="boring">unsafe { |
| </span>match *ptr { |
| _a => println!("Unreachable!"), |
| } |
| <span class="boring">} |
| </span><span class="boring">}</span></code></pre></pre> |
| <p>Here the binding loads the value of type <code>Void</code> from the <code>*ptr</code> place. In this example, this causes |
| UB since the data is not valid. In the general case, this asserts validity of the data at <code>*ptr</code>. |
| Either way, this arm will never be taken.</p> |
| <p>Finally, let's consider the empty match <code>match *ptr {}</code>. If we consider this exhaustive, then |
| having invalid data at <code>*ptr</code> is invalid. In other words, the empty match is semantically |
| equivalent to the <code>_a => ...</code> match. In the interest of explicitness, we prefer the case with an |
| arm, hence we won't tell the user to remove the <code>_a</code> arm. In other words, the <code>_a</code> arm is |
| unreachable yet not redundant. This is why we lint on redundant arms rather than unreachable |
| arms, despite the fact that the lint says "unreachable".</p> |
| <p>These considerations only affects certain places, namely those that can contain non-valid data |
| without UB. These are: pointer dereferences, reference dereferences, and union field accesses. We |
| track during exhaustiveness checking whether a given place is known to contain valid data.</p> |
| <p>Having said all that, the current implementation of exhaustiveness checking does not follow the |
| above considerations. On stable, empty types are for the most part treated as non-empty. The |
| <a href="https://github.com/rust-lang/rust/issues/51085"><code>exhaustive_patterns</code></a> feature errs on the other end: it allows omitting arms that could be |
| reachable in unsafe situations. The <a href="https://github.com/rust-lang/rust/issues/118155"><code>never_patterns</code></a> experimental feature aims to fix this and |
| permit the correct behavior of empty types in patterns.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="unsafety-checking"><a class="header" href="#unsafety-checking">Unsafety checking</a></h1> |
| <p>Certain expressions in Rust can violate memory safety and as such need to be |
| inside an <code>unsafe</code> block or function. The compiler will also warn if an unsafe |
| block is used without any corresponding unsafe operations.</p> |
| <h2 id="overview-7"><a class="header" href="#overview-7">Overview</a></h2> |
| <p>The unsafety check is located in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/check_unsafety/index.html"><code>check_unsafety</code></a> module. It performs a |
| walk over the <a href="./thir.html">THIR</a> of a function and all of its closures and inline constants. |
| It keeps track of the unsafe context: whether it has entered an <code>unsafe</code> block. |
| If an unsafe operation is used outside of an <code>unsafe</code> block, then an error is |
| reported. If an unsafe operation is used in an unsafe block then that block is |
| marked as used for <a href="unsafety-checking.html#the-unused_unsafe-lint">the unused_unsafe lint</a>.</p> |
| <p>The unsafety check needs type information so could potentially be done on the |
| HIR, making use of typeck results, THIR or MIR. THIR is chosen because there are |
| fewer cases to consider than in HIR, for example unsafe function calls and |
| unsafe method calls have the same representation in THIR. The check is not done |
| on MIR because safety checks do not depend on control flow so MIR is not |
| necessary to use and MIR doesn't have as precise spans for some expressions.</p> |
| <p>Most unsafe operations can be identified by checking the <code>ExprKind</code> in THIR and |
| checking the type of the argument. For example, dereferences of a raw pointer |
| correspond to <code>ExprKind::Deref</code>s with an argument that has a raw pointer type.</p> |
| <p>Looking for unsafe Union field accesses is a bit more complex because writing to |
| a field of a union is safe. The checker tracks when it's visiting the left-hand |
| side of an assignment expression and allows union fields to directly appear |
| there, while erroring in all other cases. Union field accesses can also occur |
| in patterns, so those have to be walked as well.</p> |
| <p>The other complicated safety check is for writes to fields of layout constrained |
| structs (such as <a href="https://doc.rust-lang.org/std/ptr/struct.NonNull.html"><code>NonNull</code></a>). These are found by looking for the borrow or |
| assignment expression and then visiting the subexpression being borrowed or |
| assigned with a separate visitor.</p> |
| <h2 id="the-unused_unsafe-lint"><a class="header" href="#the-unused_unsafe-lint">The unused_unsafe lint</a></h2> |
| <p>The unused_unsafe lint reports <code>unsafe</code> blocks that can be removed. The unsafety |
| checker records whenever it finds an operation that requires unsafe. The lint is |
| then reported if either:</p> |
| <ul> |
| <li>An <code>unsafe</code> block contains no unsafe operations</li> |
| <li>An <code>unsafe</code> block is within another unsafe block, and the outer block |
| isn't considered unused</li> |
| </ul> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span>#![deny(unused_unsafe)] |
| <span class="boring">fn main() { |
| </span>let y = 0; |
| let x: *const u8 = core::ptr::addr_of!(y); |
| unsafe { // lint reported for this block |
| unsafe { |
| let z = *x; |
| } |
| let safe_expr = 123; |
| } |
| unsafe { |
| unsafe { // lint reported for this block |
| let z = *x; |
| } |
| let unsafe_expr = *x; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="other-checks-involving-unsafe"><a class="header" href="#other-checks-involving-unsafe">Other checks involving <code>unsafe</code></a></h2> |
| <p><a href="https://doc.rust-lang.org/reference/items/traits.html#unsafe-traits">Unsafe traits</a> require an <code>unsafe impl</code> to be implemented, the check for this |
| is done as part of <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_hir_analysis/src/coherence/unsafety.rs">coherence</a>. The <code>unsafe_code</code> lint is run as a lint pass on |
| the ast that searches for unsafe blocks, functions and implementations, as well |
| as certain unsafe attributes.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="dataflow-analysis"><a class="header" href="#dataflow-analysis">Dataflow Analysis</a></h1> |
| <p>If you work on the MIR, you will frequently come across various flavors of |
| <a href="https://en.wikipedia.org/wiki/Data-flow_analysis#Basic_principles">dataflow analysis</a>. <code>rustc</code> uses dataflow to find uninitialized |
| variables, determine what variables are live across a generator <code>yield</code> |
| statement, and compute which <code>Place</code>s are borrowed at a given point in the |
| control-flow graph. Dataflow analysis is a fundamental concept in modern |
| compilers, and knowledge of the subject will be helpful to prospective |
| contributors.</p> |
| <p>However, this documentation is not a general introduction to dataflow analysis. |
| It is merely a description of the framework used to define these analyses in |
| <code>rustc</code>. It assumes that the reader is familiar with the core ideas as well as |
| some basic terminology, such as "transfer function", "fixpoint" and "lattice". |
| If you're unfamiliar with these terms, or if you want a quick refresher, |
| <a href="https://cs.au.dk/~amoeller/spa/"><em>Static Program Analysis</em></a> by Anders Møller and Michael I. Schwartzbach is an |
| excellent, freely available textbook. For those who prefer audiovisual |
| learning, we previously recommended a series of short lectures |
| by the Goethe University Frankfurt on YouTube, but it has since been deleted. |
| See <a href="https://github.com/rust-lang/rustc-dev-guide/pull/1295">this PR</a> for the context and <a href="https://github.com/rust-lang/rustc-dev-guide/pull/1295#issuecomment-1118131294">this comment</a> |
| for the alternative lectures.</p> |
| <h2 id="defining-a-dataflow-analysis"><a class="header" href="#defining-a-dataflow-analysis">Defining a Dataflow Analysis</a></h2> |
| <p>A dataflow analysis is defined by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/trait.Analysis.html"><code>Analysis</code></a> trait. In addition to the |
| type of the dataflow state, this trait defines the initial value of that state |
| at entry to each block, as well as the direction of the analysis, either |
| forward or backward. The domain of your dataflow analysis must be a <a href="https://en.wikipedia.org/wiki/Lattice_(order)">lattice</a> |
| (strictly speaking a join-semilattice) with a well-behaved <code>join</code> operator. See |
| documentation for the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/lattice/index.html"><code>lattice</code></a> module, as well as the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/lattice/trait.JoinSemiLattice.html"><code>JoinSemiLattice</code></a> |
| trait, for more information.</p> |
| <h3 id="transfer-functions-and-effects"><a class="header" href="#transfer-functions-and-effects">Transfer Functions and Effects</a></h3> |
| <p>The dataflow framework in <code>rustc</code> allows each statement (and terminator) inside |
| a basic block to define its own transfer function. For brevity, these |
| individual transfer functions are known as "effects". Each effect is applied |
| successively in dataflow order, and together they define the transfer function |
| for the entire basic block. It's also possible to define an effect for |
| particular outgoing edges of some terminators (e.g. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/trait.Analysis.html#tymethod.apply_call_return_effect"><code>apply_call_return_effect</code></a> for the <code>success</code> edge of a <code>Call</code> |
| terminator). Collectively, these are referred to as "per-edge effects".</p> |
| <h3 id="before-effects"><a class="header" href="#before-effects">"Before" Effects</a></h3> |
| <p>Observant readers of the documentation may notice that there are actually <em>two</em> |
| possible effects for each statement and terminator, the "before" effect and the |
| unprefixed (or "primary") effect. The "before" effects are applied immediately |
| before the unprefixed effect <strong>regardless of the direction of the analysis</strong>. |
| In other words, a backward analysis will apply the "before" effect and then the |
| "primary" effect when computing the transfer function for a basic block, just |
| like a forward analysis.</p> |
| <p>The vast majority of analyses should use only the unprefixed effects: Having |
| multiple effects for each statement makes it difficult for consumers to know |
| where they should be looking. However, the "before" variants can be useful in |
| some scenarios, such as when the effect of the right-hand side of an assignment |
| statement must be considered separately from the left-hand side.</p> |
| <h3 id="convergence"><a class="header" href="#convergence">Convergence</a></h3> |
| <p>Your analysis must converge to "fixpoint", otherwise it will run forever. |
| Converging to fixpoint is just another way of saying "reaching equilibrium". |
| In order to reach equilibrium, your analysis must obey some laws. One of the |
| laws it must obey is that the bottom value<sup class="footnote-reference" id="fr-bottom-purpose-1"><a href="#footnote-bottom-purpose">1</a></sup> joined with some |
| other value equals the second value. Or, as an equation:</p> |
| <blockquote> |
| <p><em>bottom</em> join <em>x</em> = <em>x</em></p> |
| </blockquote> |
| <p>Another law is that your analysis must have a "top value" such that</p> |
| <blockquote> |
| <p><em>top</em> join <em>x</em> = <em>top</em></p> |
| </blockquote> |
| <p>Having a top value ensures that your semilattice has a finite height, and the |
| law state above ensures that once the dataflow state reaches top, it will no |
| longer change (the fixpoint will be top).</p> |
| <h2 id="a-brief-example"><a class="header" href="#a-brief-example">A Brief Example</a></h2> |
| <p>This section provides a brief example of a simple data-flow analysis at a high |
| level. It doesn't explain everything you need to know, but hopefully it will |
| make the rest of this page clearer.</p> |
| <p>Let's say we want to do a simple analysis to find if <code>mem::transmute</code> may have |
| been called by a certain point in the program. Our analysis domain will just |
| be a <code>bool</code> that records whether <code>transmute</code> has been called so far. The bottom |
| value will be <code>false</code>, since by default <code>transmute</code> has not been called. The top |
| value will be <code>true</code>, since our analysis is done as soon as we determine that |
| <code>transmute</code> has been called. Our join operator will just be the boolean OR (<code>||</code>) |
| operator. We use OR and not AND because of this case:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span><span class="boring">unsafe fn example(some_cond: bool) { |
| </span>let x = if some_cond { |
| std::mem::transmute::<i32, u32>(0_i32) // transmute was called! |
| } else { |
| 1_u32 // transmute was not called |
| }; |
| |
| // Has transmute been called by this point? We conservatively approximate that |
| // as yes, and that is why we use the OR operator. |
| println!("x: {}", x); |
| <span class="boring">} |
| </span><span class="boring">}</span></code></pre></pre> |
| <h2 id="inspecting-the-results-of-a-dataflow-analysis"><a class="header" href="#inspecting-the-results-of-a-dataflow-analysis">Inspecting the Results of a Dataflow Analysis</a></h2> |
| <p>Once you have constructed an analysis, you must call <code>iterate_to_fixpoint</code> |
| which will return a <code>Results</code>, which contains the dataflow state at fixpoint |
| upon entry of each block. Once you have a <code>Results</code>, you can inspect the |
| dataflow state at fixpoint at any point in the CFG. If you only need the state |
| at a few locations (e.g., each <code>Drop</code> terminator) use a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/struct.ResultsCursor.html"><code>ResultsCursor</code></a>. If |
| you need the state at <em>every</em> location, a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/trait.ResultsVisitor.html"><code>ResultsVisitor</code></a> will be more |
| efficient.</p> |
| <pre><code class="language-text"> Analysis |
| | |
| | iterate_to_fixpoint() |
| | |
| Results |
| / \ |
| into_results_cursor(…) / \ visit_with(…) |
| / \ |
| ResultsCursor ResultsVisitor |
| </code></pre> |
| <p>For example, the following code uses a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/trait.ResultsVisitor.html"><code>ResultsVisitor</code></a>...</p> |
| <pre><code class="language-rust ignore">// Assuming `MyVisitor` implements `ResultsVisitor<FlowState = MyAnalysis::Domain>`... |
| let mut my_visitor = MyVisitor::new(); |
| |
| // inspect the fixpoint state for every location within every block in RPO. |
| let results = MyAnalysis::new() |
| .iterate_to_fixpoint(tcx, body, None); |
| results.visit_with(body, &mut my_visitor);`</code></pre> |
| <p>whereas this code uses <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/struct.ResultsCursor.html"><code>ResultsCursor</code></a>:</p> |
| <pre><code class="language-rust ignore">let mut results = MyAnalysis::new() |
| .iterate_to_fixpoint(tcx, body, None); |
| .into_results_cursor(body); |
| |
| // Inspect the fixpoint state immediately before each `Drop` terminator. |
| for (bb, block) in body.basic_blocks().iter_enumerated() { |
| if let TerminatorKind::Drop { .. } = block.terminator().kind { |
| results.seek_before_primary_effect(body.terminator_loc(bb)); |
| let state = results.get(); |
| println!("state before drop: {:#?}", state); |
| } |
| }</code></pre> |
| <h3 id="graphviz-diagrams"><a class="header" href="#graphviz-diagrams">Graphviz Diagrams</a></h3> |
| <p>When the results of a dataflow analysis are not what you expect, it often helps |
| to visualize them. This can be done with the <code>-Z dump-mir</code> flags described in |
| <a href="mir/./debugging.html">Debugging MIR</a>. Start with <code>-Z dump-mir=F -Z dump-mir-dataflow</code>, where <code>F</code> is |
| either "all" or the name of the MIR body you are interested in.</p> |
| <p>These <code>.dot</code> files will be saved in your <code>mir_dump</code> directory and will have the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/trait.Analysis.html#associatedconstant.NAME"><code>NAME</code></a> of the analysis (e.g. <code>maybe_inits</code>) as part of their filename. Each |
| visualization will display the full dataflow state at entry and exit of each |
| block, as well as any changes that occur in each statement and terminator. See |
| the example below:</p> |
| <p><img src="mir/../img/dataflow-graphviz-example.png" alt="A graphviz diagram for a dataflow analysis" /></p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-bottom-purpose"> |
| <p>The bottom value's primary purpose is as the initial dataflow |
| state. Each basic block's entry state is initialized to bottom before the |
| analysis starts. <a href="#fr-bottom-purpose-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="drop-elaboration"><a class="header" href="#drop-elaboration">Drop elaboration</a></h1> |
| <h2 id="dynamic-drops"><a class="header" href="#dynamic-drops">Dynamic drops</a></h2> |
| <p>According to the <a href="https://doc.rust-lang.org/reference/destructors.html">reference</a>:</p> |
| <blockquote> |
| <p>When an initialized variable or temporary goes out of scope, its destructor |
| is run, or it is dropped. Assignment also runs the destructor of its |
| left-hand operand, if it's initialized. If a variable has been partially |
| initialized, only its initialized fields are dropped.</p> |
| </blockquote> |
| <p>When building the MIR, the <code>Drop</code> and <code>DropAndReplace</code> terminators represent |
| places where drops may occur. However, in this phase, the presence of these |
| terminators does not guarantee that a destructor will run. That's because the |
| target of a drop may be uninitialized (usually because it has been moved from) |
| before the terminator is reached. In general, we cannot know at compile-time whether a |
| variable is initialized.</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>let mut y = vec![]; |
| |
| { |
| let x = vec![1, 2, 3]; |
| if std::process::id() % 2 == 0 { |
| y = x; // conditionally move `x` into `y` |
| } |
| } // `x` goes out of scope here. Should it be dropped? |
| <span class="boring">}</span></code></pre></pre> |
| <p>In these cases, we need to keep track of whether a variable is initialized |
| <em>dynamically</em>. The rules are laid out in detail in <a href="https://rust-lang.github.io/rfcs/0320-nonzeroing-dynamic-drop.html">RFC 320: Non-zeroing |
| dynamic drops</a>.</p> |
| <h2 id="drop-obligations"><a class="header" href="#drop-obligations">Drop obligations</a></h2> |
| <p>From the RFC:</p> |
| <blockquote> |
| <p>When a local variable becomes initialized, it establishes a set of "drop |
| obligations": a set of structural paths (e.g. a local <code>a</code>, or a path to a |
| field <code>b.f.y</code>) that need to be dropped.</p> |
| <p>The drop obligations for a local variable x of struct-type <code>T</code> are computed |
| from analyzing the structure of <code>T</code>. If <code>T</code> itself implements <code>Drop</code>, then <code>x</code> is a |
| drop obligation. If <code>T</code> does not implement <code>Drop</code>, then the set of drop |
| obligations is the union of the drop obligations of the fields of <code>T</code>.</p> |
| </blockquote> |
| <p>When a structural path is moved from (and thus becomes uninitialized), any drop |
| obligations for that path or its descendants (<code>path.f</code>, <code>path.f.g.h</code>, etc.) are |
| released. Types with <code>Drop</code> implementations do not permit moves from individual |
| fields, so there is no need to track initializedness through them.</p> |
| <p>When a local variable goes out of scope (<code>Drop</code>), or when a structural path is |
| overwritten via assignment (<code>DropAndReplace</code>), we check for any drop |
| obligations for that variable or path. Unless that obligation has been |
| released by this point, its associated <code>Drop</code> implementation will be called. |
| For <code>enum</code> types, only fields corresponding to the "active" variant need to be |
| dropped. When processing drop obligations for such types, we first check the |
| discriminant to determine the active variant. All drop obligations for variants |
| besides the active one are ignored.</p> |
| <p>Here are a few interesting types to help illustrate these rules:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct NoDrop(u8); // No `Drop` impl. No fields with `Drop` impls. |
| |
| struct NeedsDrop(Vec<u8>); // No `Drop` impl but has fields with `Drop` impls. |
| |
| struct ThinVec(*const u8); // Custom `Drop` impl. Individual fields cannot be moved from. |
| |
| impl Drop for ThinVec { |
| fn drop(&mut self) { /* ... */ } |
| } |
| |
| enum MaybeDrop { |
| Yes(NeedsDrop), |
| No(NoDrop), |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="drop-elaboration-1"><a class="header" href="#drop-elaboration-1">Drop elaboration</a></h2> |
| <p>One valid model for these rules is to keep a boolean flag (a "drop flag") for |
| every structural path that is used at any point in the function. This flag is |
| set when its path is initialized and is cleared when the path is moved from. |
| When a <code>Drop</code> occurs, we check the flags for every obligation associated with |
| the target of the <code>Drop</code> and call the associated <code>Drop</code> impl for those that are |
| still applicable.</p> |
| <p>This process—transforming the newly built MIR with its imprecise <code>Drop</code> and |
| <code>DropAndReplace</code> terminators into one with drop flags—is known as drop |
| elaboration. When a MIR statement causes a variable to become initialized (or |
| uninitialized), drop elaboration inserts code that sets (or clears) the drop |
| flag for that variable. It wraps <code>Drop</code> terminators in conditionals that check |
| the newly inserted drop flags.</p> |
| <p>Drop elaboration also splits <code>DropAndReplace</code> terminators into a <code>Drop</code> of the |
| target and a write of the newly dropped place. This is somewhat unrelated to what |
| we've discussed above.</p> |
| <p>Once this is complete, <code>Drop</code> terminators in the MIR correspond to a call to |
| the "drop glue" or "drop shim" for the type of the dropped place. The drop |
| glue for a type calls the <code>Drop</code> impl for that type (if one exists), and then |
| recursively calls the drop glue for all fields of that type.</p> |
| <h2 id="drop-elaboration-in-rustc"><a class="header" href="#drop-elaboration-in-rustc">Drop elaboration in <code>rustc</code></a></h2> |
| <p>The approach described above is more expensive than necessary. One can imagine |
| a few optimizations:</p> |
| <ul> |
| <li>Only paths that are the target of a <code>Drop</code> (or have the target as a prefix) |
| need drop flags.</li> |
| <li>Some variables are known to be initialized (or uninitialized) when they are |
| dropped. These do not need drop flags.</li> |
| <li>If a set of paths are only dropped or moved from via a shared prefix, those |
| paths can share a single drop flag.</li> |
| </ul> |
| <p>A subset of these are implemented in <code>rustc</code>.</p> |
| <p>In the compiler, drop elaboration is split across several modules. The pass |
| itself is defined <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_mir_transform/src/elaborate_drops.rs">here</a>, but the <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_mir_dataflow/src/elaborate_drops.rs">main logic</a> is |
| defined elsewhere since it is also used to build <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_mir_transform/src/shim.rs">drop shims</a>.</p> |
| <p>Drop elaboration designates each <code>Drop</code> in the newly built MIR as one of four |
| kinds:</p> |
| <ul> |
| <li><code>Static</code>, the target is always initialized.</li> |
| <li><code>Dead</code>, the target is always <strong>un</strong>initialized.</li> |
| <li><code>Conditional</code>, the target is either wholly initialized or wholly |
| uninitialized. It is not partly initialized.</li> |
| <li><code>Open</code>, the target may be partly initialized.</li> |
| </ul> |
| <p>For this, it uses a pair of dataflow analyses, <code>MaybeInitializedPlaces</code> and |
| <code>MaybeUninitializedPlaces</code>. If a place is in one but not the other, then the |
| initializedness of the target is known at compile-time (<code>Dead</code> or <code>Static</code>). |
| In this case, drop elaboration does not add a flag for the target. It simply |
| removes (<code>Dead</code>) or preserves (<code>Static</code>) the <code>Drop</code> terminator.</p> |
| <p>For <code>Conditional</code> drops, we know that the initializedness of the variable as a |
| whole is the same as the initializedness of its fields. Therefore, once we |
| generate a drop flag for the target of that drop, it's safe to call the drop |
| glue for that target.</p> |
| <h3 id="open-drops"><a class="header" href="#open-drops"><code>Open</code> drops</a></h3> |
| <p><code>Open</code> drops are the most complex, since we need to break down a single <code>Drop</code> |
| terminator into several different ones, one for each field of the target whose |
| type has drop glue (<code>Ty::needs_drop</code>). We cannot call the drop glue for the |
| target itself because that requires all fields of the target to be initialized. |
| Remember, variables whose type has a custom <code>Drop</code> impl do not allow <code>Open</code> |
| drops because their fields cannot be moved from.</p> |
| <p>This is accomplished by recursively categorizing each field as <code>Dead</code>, |
| <code>Static</code>, <code>Conditional</code> or <code>Open</code>. Fields whose type does not have drop glue |
| are automatically <code>Dead</code> and need not be considered during the recursion. When |
| we reach a field whose kind is not <code>Open</code>, we handle it as we did above. If the |
| field is also <code>Open</code>, the recursion continues.</p> |
| <p>It's worth noting how we handle <code>Open</code> drops of enums. Inside drop elaboration, |
| each variant of the enum is treated like a field, with the invariant that only |
| one of those "variant fields" can be initialized at any given time. In the |
| general case, we do not know which variant is the active one, so we will have |
| to call the drop glue for the enum (which checks the discriminant) or check the |
| discriminant ourselves as part of an elaborated <code>Open</code> drop. However, in |
| certain cases (within a <code>match</code> arm, for example) we do know which variant of |
| an enum is active. This information is encoded in the <code>MaybeInitializedPlaces</code> |
| and <code>MaybeUninitializedPlaces</code> dataflow analyses by marking all places |
| corresponding to inactive variants as uninitialized.</p> |
| <h3 id="cleanup-paths"><a class="header" href="#cleanup-paths">Cleanup paths</a></h3> |
| <p>TODO: Discuss drop elaboration and unwinding.</p> |
| <h2 id="aside-drop-elaboration-and-const-eval"><a class="header" href="#aside-drop-elaboration-and-const-eval">Aside: drop elaboration and const-eval</a></h2> |
| <p>In Rust, functions that are eligible for evaluation at compile-time must be |
| marked explicitly using the <code>const</code> keyword. This includes implementations of |
| the <code>Drop</code> trait, which may or may not be <code>const</code>. Code that is eligible for |
| compile-time evaluation may only call <code>const</code> functions, so any calls to |
| non-const <code>Drop</code> implementations in such code must be forbidden.</p> |
| <p>A call to a <code>Drop</code> impl is encoded as a <code>Drop</code> terminator in the MIR. However, |
| as we discussed above, a <code>Drop</code> terminator in newly built MIR does not |
| necessarily result in a call to <code>Drop::drop</code>. The drop target may be |
| uninitialized at that point. This means that checking for non-const <code>Drop</code>s on |
| the newly built MIR can result in spurious errors. Instead, we wait until after |
| drop elaboration runs, which eliminates <code>Dead</code> drops (ones where the target is |
| known to be uninitialized) to run these checks.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mir-borrow-check"><a class="header" href="#mir-borrow-check">MIR borrow check</a></h1> |
| <p>The borrow check is Rust's "secret sauce" – it is tasked with |
| enforcing a number of properties:</p> |
| <ul> |
| <li>That all variables are initialized before they are used.</li> |
| <li>That you can't move the same value twice.</li> |
| <li>That you can't move a value while it is borrowed.</li> |
| <li>That you can't access a place while it is mutably borrowed (except through |
| the reference).</li> |
| <li>That you can't mutate a place while it is immutably borrowed.</li> |
| <li>etc</li> |
| </ul> |
| <p>The borrow checker operates on the MIR. An older implementation operated on the |
| HIR. Doing borrow checking on MIR has several advantages:</p> |
| <ul> |
| <li>The MIR is <em>far</em> less complex than the HIR; the radical desugaring |
| helps prevent bugs in the borrow checker. (If you're curious, you |
| can see |
| <a href="https://github.com/rust-lang/rust/issues/47366">a list of bugs that the MIR-based borrow checker fixes here</a>.)</li> |
| <li>Even more importantly, using the MIR enables <a href="https://rust-lang.github.io/rfcs/2094-nll.html">"non-lexical lifetimes"</a>, |
| which are regions derived from the control-flow graph.</li> |
| </ul> |
| <h3 id="major-phases-of-the-borrow-checker"><a class="header" href="#major-phases-of-the-borrow-checker">Major phases of the borrow checker</a></h3> |
| <p>The borrow checker source is found in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/index.html">the <code>rustc_borrowck</code> crate</a>. The main entry point is |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/fn.mir_borrowck.html"><code>mir_borrowck</code></a> query.</p> |
| <ul> |
| <li>We first create a <strong>local copy</strong> of the MIR. In the coming steps, |
| we will modify this copy in place to modify the types and things to |
| include references to the new regions that we are computing.</li> |
| <li>We then invoke <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.replace_regions_in_mir.html"><code>replace_regions_in_mir</code></a> to modify our local MIR. |
| Among other things, this function will replace all of the <a href="./appendix/glossary.html#region">regions</a> |
| in the MIR with fresh <a href="./appendix/glossary.html#inf-var">inference variables</a>.</li> |
| <li>Next, we perform a number of |
| <a href="./appendix/background.html#dataflow">dataflow analyses</a> that |
| compute what data is moved and when.</li> |
| <li>We then do a <a href="borrow_check/type_check.html">second type check</a> across the MIR: |
| the purpose of this type check is to determine all of the constraints between |
| different regions.</li> |
| <li>Next, we do <a href="borrow_check/region_inference.html">region inference</a>, which computes |
| the values of each region — basically, the points in the control-flow graph where |
| each lifetime must be valid according to the constraints we collected.</li> |
| <li>At this point, we can compute the "borrows in scope" at each point.</li> |
| <li>Finally, we do a second walk over the MIR, looking at the actions it |
| does and reporting errors. For example, if we see a statement like |
| <code>*a + 1</code>, then we would check that the variable <code>a</code> is initialized |
| and that it is not mutably borrowed, as either of those would |
| require an error to be reported. Doing this check requires the results of all |
| the previous analyses.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="tracking-moves-and-initialization"><a class="header" href="#tracking-moves-and-initialization">Tracking moves and initialization</a></h1> |
| <p>Part of the borrow checker's job is to track which variables are |
| "initialized" at any given point in time -- this also requires |
| figuring out where moves occur and tracking those.</p> |
| <h2 id="initialization-and-moves"><a class="header" href="#initialization-and-moves">Initialization and moves</a></h2> |
| <p>From a user's perspective, initialization -- giving a variable some |
| value -- and moves -- transferring ownership to another place -- might |
| seem like distinct topics. Indeed, our borrow checker error messages |
| often talk about them differently. But <strong>within the borrow checker</strong>, |
| they are not nearly as separate. Roughly speaking, the borrow checker |
| tracks the set of "initialized places" at any point in the source |
| code. Assigning to a previously uninitialized local variable adds it |
| to that set; moving from a local variable removes it from that set.</p> |
| <p>Consider this example:</p> |
| <pre><code class="language-rust ignore">fn foo() { |
| let a: Vec<u32>; |
| |
| // a is not initialized yet |
| |
| a = vec![22]; |
| |
| // a is initialized here |
| |
| std::mem::drop(a); // a is moved here |
| |
| // a is no longer initialized here |
| |
| let l = a.len(); //~ ERROR |
| }</code></pre> |
| <p>Here you can see that <code>a</code> starts off as uninitialized; once it is |
| assigned, it becomes initialized. But when <code>drop(a)</code> is called, that |
| moves <code>a</code> into the call, and hence it becomes uninitialized again.</p> |
| <h2 id="subsections"><a class="header" href="#subsections">Subsections</a></h2> |
| <p>To make it easier to peruse, this section is broken into a number of |
| subsections:</p> |
| <ul> |
| <li><a href="borrow_check/./moves_and_initialization/move_paths.html">Move paths</a> the |
| <em>move path</em> concept that we use to track which local variables (or parts of |
| local variables, in some cases) are initialized.</li> |
| <li>TODO <em>Rest not yet written</em> =)</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="move-paths"><a class="header" href="#move-paths">Move paths</a></h1> |
| <p>In reality, it's not enough to track initialization at the granularity |
| of local variables. Rust also allows us to do moves and initialization |
| at the field granularity:</p> |
| <pre><code class="language-rust ignore">fn foo() { |
| let a: (Vec<u32>, Vec<u32>) = (vec![22], vec![44]); |
| |
| // a.0 and a.1 are both initialized |
| |
| let b = a.0; // moves a.0 |
| |
| // a.0 is not initialized, but a.1 still is |
| |
| let c = a.0; // ERROR |
| let d = a.1; // OK |
| }</code></pre> |
| <p>To handle this, we track initialization at the granularity of a <strong>move |
| path</strong>. A <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePath.html"><code>MovePath</code></a> represents some location that the user can |
| initialize, move, etc. So e.g. there is a move-path representing the |
| local variable <code>a</code>, and there is a move-path representing <code>a.0</code>. Move |
| paths roughly correspond to the concept of a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a> from MIR, but |
| they are indexed in ways that enable us to do move analysis more |
| efficiently.</p> |
| <h2 id="move-path-indices"><a class="header" href="#move-path-indices">Move path indices</a></h2> |
| <p>Although there is a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePath.html"><code>MovePath</code></a> data structure, they are never referenced |
| directly. Instead, all the code passes around <em>indices</em> of type |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a>. If you need to get information about a move path, you use |
| this index with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MoveData.html#structfield.move_paths"><code>move_paths</code> field of the <code>MoveData</code></a>. For |
| example, to convert a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a> <code>mpi</code> into a MIR <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a>, you might |
| access the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePath.html#structfield.place"><code>MovePath::place</code></a> field like so:</p> |
| <pre><code class="language-rust ignore">move_data.move_paths[mpi].place</code></pre> |
| <h2 id="building-move-paths"><a class="header" href="#building-move-paths">Building move paths</a></h2> |
| <p>One of the first things we do in the MIR borrow check is to construct |
| the set of move paths. This is done as part of the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MoveData.html#method.gather_moves"><code>MoveData::gather_moves</code></a> function. This function uses a MIR visitor |
| called <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/builder/struct.MoveDataBuilder.html"><code>MoveDataBuilder</code></a> to walk the MIR and look at how each <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a> |
| within is accessed. For each such <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a>, it constructs a |
| corresponding <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a>. It also records when/where that |
| particular move path is moved/initialized, but we'll get to that in a |
| later section.</p> |
| <h3 id="illegal-move-paths"><a class="header" href="#illegal-move-paths">Illegal move paths</a></h3> |
| <p>We don't actually create a move-path for <strong>every</strong> <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a> that gets |
| used. In particular, if it is illegal to move from a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a>, then |
| there is no need for a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a>. Some examples:</p> |
| <ul> |
| <li>You cannot move an individual element of an array, so if we have e.g. <code>foo: [String; 3]</code>, |
| there would be no move-path for <code>foo[1]</code>.</li> |
| <li>You cannot move from inside of a borrowed reference, so if we have e.g. <code>foo: &String</code>, |
| there would be no move-path for <code>*foo</code>.</li> |
| </ul> |
| <p>These rules are enforced by the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/builder/struct.MoveDataBuilder.html#method.move_path_for"><code>move_path_for</code></a> function, which |
| converts a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a> into a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a> -- in error cases like |
| those just discussed, the function returns an <code>Err</code>. This in turn |
| means we don't have to bother tracking whether those places are |
| initialized (which lowers overhead).</p> |
| <h2 id="projections"><a class="header" href="#projections">Projections</a></h2> |
| <p>Instead of using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/type.PlaceElem.html"><code>PlaceElem</code></a>, projections in move paths are stored as <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/enum.MoveSubPath.html"><code>MoveSubPath</code></a>s. |
| Projections that can't be moved out of and projections that can be skipped are not represented.</p> |
| <p>Subslice projections of arrays (produced by slice patterns) are special; they're turned into |
| multiple <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/enum.MoveSubPath.html#variant.ConstantIndex"><code>ConstantIndex</code></a> subpaths, one for each element in the subslice.</p> |
| <h2 id="looking-up-a-move-path"><a class="header" href="#looking-up-a-move-path">Looking up a move-path</a></h2> |
| <p>If you have a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a> and you would like to convert it to a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a>, you |
| can do that using the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathLookup.html"><code>MovePathLookup</code></a> structure found in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MoveData.html#structfield.rev_lookup"><code>rev_lookup</code></a> field |
| of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MoveData.html"><code>MoveData</code></a>. There are two different methods:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathLookup.html#method.find_local"><code>find_local</code></a>, which takes a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Local.html"><code>mir::Local</code></a> representing a local |
| variable. This is the easier method, because we <strong>always</strong> create a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a> for every local variable.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathLookup.html#method.find"><code>find</code></a>, which takes an arbitrary <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a>. This method is a bit |
| more annoying to use, precisely because we don't have a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a> for <strong>every</strong> <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html"><code>Place</code></a> (as we just discussed in |
| the "illegal move paths" section). Therefore, <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathLookup.html#method.find"><code>find</code></a> returns a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/enum.LookupResult.html"><code>LookupResult</code></a> indicating the closest path it was able to find |
| that exists (e.g., for <code>foo[1]</code>, it might return just the path for |
| <code>foo</code>).</li> |
| </ul> |
| <h2 id="cross-references"><a class="header" href="#cross-references">Cross-references</a></h2> |
| <p>As we noted above, move-paths are stored in a big vector and |
| referenced via their <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a>. However, within this vector, |
| they are also structured into a tree. So for example if you have the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MovePathIndex.html"><code>MovePathIndex</code></a> for <code>a.b.c</code>, you can go to its parent move-path |
| <code>a.b</code>. You can also iterate over all children paths: so, from <code>a.b</code>, |
| you might iterate to find the path <code>a.b.c</code> (here you are iterating |
| just over the paths that are <strong>actually referenced</strong> in the source, |
| not all <strong>possible</strong> paths that could have been referenced). These |
| references are used for example in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_dataflow/move_paths/struct.MoveData.html#method.find_in_move_path_or_its_descendants"><code>find_in_move_path_or_its_descendants</code></a> function, which determines |
| whether a move-path (e.g., <code>a.b</code>) or any child of that move-path |
| (e.g.,<code>a.b.c</code>) matches a given predicate.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="the-mir-type-check"><a class="header" href="#the-mir-type-check">The MIR type-check</a></h1> |
| <p>A key component of the borrow check is the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/index.html">MIR type-check</a>. |
| This check walks the MIR and does a complete "type check" -- the same |
| kind you might find in any other language. In the process of doing |
| this type-check, we also uncover the region constraints that apply to |
| the program.</p> |
| <p>TODO -- elaborate further? Maybe? :)</p> |
| <h2 id="user-types"><a class="header" href="#user-types">User types</a></h2> |
| <p>At the start of MIR type-check, we replace all regions in the body with new unconstrained regions. |
| However, this would cause us to accept the following program:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a>(x: &'a u32) { |
| let y: &'static u32 = x; |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>By erasing the lifetimes in the type of <code>y</code> we no longer know that it is supposed to be <code>'static</code>, |
| ignoring the intentions of the user.</p> |
| <p>To deal with this we remember all places where the user explicitly mentioned a type during |
| HIR type-check as <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.CanonicalUserTypeAnnotation.html"><code>CanonicalUserTypeAnnotations</code></a>.</p> |
| <p>There are two different annotations we care about:</p> |
| <ul> |
| <li>explicit type ascriptions, e.g. <code>let y: &'static u32</code> results in <code>UserType::Ty(&'static u32)</code>.</li> |
| <li>explicit generic arguments, e.g. <code>x.foo<&'a u32, Vec<String>></code> |
| results in <code>UserType::TypeOf(foo_def_id, [&'a u32, Vec<String>])</code>.</li> |
| </ul> |
| <p>As we do not want the region inference from the HIR type-check to influence MIR typeck, |
| we store the user type right after lowering it from the HIR. |
| This means that it may still contain inference variables, |
| which is why we are using <strong>canonical</strong> user type annotations. |
| We replace all inference variables with existential bound variables instead. |
| Something like <code>let x: Vec<_></code> would therefore result in <code>exists<T> UserType::Ty(Vec<T>)</code>.</p> |
| <p>A pattern like <code>let Foo(x): Foo<&'a u32></code> has a user type <code>Foo<&'a u32></code> but |
| the actual type of <code>x</code> should only be <code>&'a u32</code>. For this, we use a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.UserTypeProjection.html"><code>UserTypeProjection</code></a>.</p> |
| <p>In the MIR, we deal with user types in two slightly different ways.</p> |
| <p>Given a MIR local corresponding to a variable in a pattern which has an explicit type annotation, |
| we require the type of that local to be equal to the type of the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.UserTypeProjection.html"><code>UserTypeProjection</code></a>. |
| This is directly stored in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.LocalDecl.html"><code>LocalDecl</code></a>.</p> |
| <p>We also constrain the type of scrutinee expressions, e.g. the type of <code>x</code> in <code>let _: &'a u32 = x;</code>. |
| Here <code>T_x</code> only has to be a subtype of the user type, so we instead use |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.StatementKind.html#variant.AscribeUserType"><code>StatementKind::AscribeUserType</code></a> for that.</p> |
| <p>Note that we do not directly use the user type as the MIR typechecker |
| doesn't really deal with type and const inference variables. We instead store the final |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.CanonicalUserTypeAnnotation.html#structfield.inferred_ty"><code>inferred_type</code></a> from the HIR type-checker. During MIR typeck, we then replace its regions |
| with new nll inference vars and relate it with the actual <code>UserType</code> to get the correct region |
| constraints again.</p> |
| <p>After the MIR type-check, all user type annotations get discarded, as they aren't needed anymore.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="drop-check"><a class="header" href="#drop-check">Drop Check</a></h1> |
| <p>We generally require the type of locals to be well-formed whenever the |
| local is used. This includes proving the where-bounds of the local and |
| also requires all regions used by it to be live.</p> |
| <p>The only exception to this is when implicitly dropping values when they |
| go out of scope. This does not necessarily require the value to be live:</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let x = vec![]; |
| { |
| let y = String::from("I am temporary"); |
| x.push(&y); |
| } |
| // `x` goes out of scope here, after the reference to `y` |
| // is invalidated. This means that while dropping `x` its type |
| // is not well-formed as it contain regions which are not live. |
| }</code></pre></pre> |
| <p>This is only sound if dropping the value does not try to access any dead |
| region. We check this by requiring the type of the value to be |
| drop-live. |
| The requirements for which are computed in <code>fn dropck_outlives</code>.</p> |
| <p>The rest of this section uses the following type definition for a type |
| which requires its region parameter to be live:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>struct PrintOnDrop<'a>(&'a str); |
| impl<'a> Drop for PrintOnDrop<'_> { |
| fn drop(&mut self) { |
| println!("{}", self.0); |
| } |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <h2 id="how-values-are-dropped"><a class="header" href="#how-values-are-dropped">How values are dropped</a></h2> |
| <p>At its core, a value of type <code>T</code> is dropped by executing its "drop |
| glue". Drop glue is compiler generated and first calls <code><T as Drop>::drop</code> and then recursively calls the drop glue of any recursively |
| owned values.</p> |
| <ul> |
| <li>If <code>T</code> has an explicit <code>Drop</code> impl, call <code><T as Drop>::drop</code>.</li> |
| <li>Regardless of whether <code>T</code> implements <code>Drop</code>, recurse into all values |
| <em>owned</em> by <code>T</code>: |
| <ul> |
| <li>references, raw pointers, function pointers, function items, trait |
| objects<sup class="footnote-reference" id="fr-traitobj-1"><a href="#footnote-traitobj">1</a></sup>, and scalars do not own anything.</li> |
| <li>tuples, slices, and arrays consider their elements to be owned. |
| For arrays of length zero we do not own any value of the element |
| type.</li> |
| <li>all fields (of all variants) of ADTs are considered owned. We |
| consider all variants for enums. The exception here is |
| <code>ManuallyDrop<U></code> which is not considered to own <code>U</code>. |
| <code>PhantomData<U></code> also does not own anything. |
| closures and generators own their captured upvars.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Whether a type has drop glue is returned by <a href="https://github.com/rust-lang/rust/blob/320b412f9c55bf480d26276ff0ab480e4ecb29c0/compiler/rustc_middle/src/ty/util.rs#L1086-L1108"><code>fn Ty::needs_drop</code></a>.</p> |
| <h3 id="partially-dropping-a-local"><a class="header" href="#partially-dropping-a-local">Partially dropping a local</a></h3> |
| <p>For types which do not implement <code>Drop</code> themselves, we can also |
| partially move parts of the value before dropping the rest. In this case |
| only the drop glue for the not-yet moved values is called, e.g.</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let mut x = (PrintOnDrop("third"), PrintOnDrop("first")); |
| drop(x.1); |
| println!("second") |
| }</code></pre></pre> |
| <p>During MIR building we assume that a local may get dropped whenever it |
| goes out of scope <em>as long as its type needs drop</em>. Computing the exact |
| drop glue for a variable happens <strong>after</strong> borrowck in the |
| <code>ElaborateDrops</code> pass. This means that even if some part of the local |
| have been dropped previously, dropck still requires this value to be |
| live. This is the case even if we completely moved a local.</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let mut x; |
| { |
| let temp = String::from("I am temporary"); |
| x = PrintOnDrop(&temp); |
| drop(x); |
| } |
| } //~ ERROR `temp` does not live long enough.</code></pre></pre> |
| <p>It should be possible to add some amount of drop elaboration before |
| borrowck, allowing this example to compile. There is an unstable feature |
| to move drop elaboration before const checking: |
| <a href="https://github.com/rust-lang/rust/issues/73255">#73255</a>. Such a feature |
| gate does not exist for doing some drop elaboration before borrowck, |
| although there's a <a href="https://github.com/rust-lang/compiler-team/issues/558">relevant |
| MCP</a>.</p> |
| <h3 id="dropck_outlives"><a class="header" href="#dropck_outlives"><code>dropck_outlives</code></a></h3> |
| <p>There are two distinct "liveness" computations that we perform:</p> |
| <ul> |
| <li>a value <code>v</code> is <em>use-live</em> at location <code>L</code> if it may be "used" later; a |
| <em>use</em> here is basically anything that is not a <em>drop</em></li> |
| <li>a value <code>v</code> is <em>drop-live</em> at location <code>L</code> if it maybe dropped later</li> |
| </ul> |
| <p>When things are <em>use-live</em>, their entire type must be valid at <code>L</code>. When |
| they are <em>drop-live</em>, all regions that are required by dropck must be |
| valid at <code>L</code>. The values dropped in the MIR are <em>places</em>.</p> |
| <p>The constraints computed by <code>dropck_outlives</code> for a type closely match |
| the generated drop glue for that type. Unlike drop glue, |
| <code>dropck_outlives</code> cares about the types of owned values, not the values |
| itself. For a value of type <code>T</code></p> |
| <ul> |
| <li>if <code>T</code> has an explicit <code>Drop</code>, require all generic arguments to be |
| live, unless they are marked with <code>#[may_dangle]</code> in which case they |
| are fully ignored</li> |
| <li>regardless of whether <code>T</code> has an explicit <code>Drop</code>, recurse into all |
| types <em>owned</em> by <code>T</code> |
| <ul> |
| <li>references, raw pointers, function pointers, function items, trait |
| objects<sup class="footnote-reference" id="fr-traitobj-2"><a href="#footnote-traitobj">1</a></sup>, and scalars do not own anything.</li> |
| <li>tuples, slices and arrays consider their element type to be owned. |
| <strong>For arrays we currently do not check whether their length is |
| zero</strong>.</li> |
| <li>all fields (of all variants) of ADTs are considered owned. The |
| exception here is <code>ManuallyDrop<U></code> which is not considered to own |
| <code>U</code>. <strong>We consider <code>PhantomData<U></code> to own <code>U</code></strong>.</li> |
| <li>closures and generators own their captured upvars.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>The sections marked in bold are cases where <code>dropck_outlives</code> considers |
| types to be owned which are ignored by <code>Ty::needs_drop</code>. We only rely on |
| <code>dropck_outlives</code> if <code>Ty::needs_drop</code> for the containing local returned |
| <code>true</code>.This means liveness requirements can change depending on whether |
| a type is contained in a larger local. <strong>This is inconsistent, and |
| should be fixed: an example <a href="https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=8b5f5f005a03971b22edb1c20c5e6cbe">for |
| arrays</a> |
| and <a href="https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=44c6e2b1fae826329fd54c347603b6c8">for |
| <code>PhantomData</code></a>.</strong><sup class="footnote-reference" id="fr-core-1"><a href="#footnote-core">2</a></sup></p> |
| <p>One possible way these inconsistencies can be fixed is by MIR building |
| to be more pessimistic, probably by making <code>Ty::needs_drop</code> weaker, or |
| alternatively, changing <code>dropck_outlives</code> to be more precise, requiring |
| fewer regions to be live.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-traitobj"> |
| <p>you can consider trait objects to have a builtin <code>Drop</code> |
| implementation which directly uses the <code>drop_in_place</code> provided by the |
| vtable. This <code>Drop</code> implementation requires all its generic parameters |
| to be live. <a href="#fr-traitobj-1">↩</a> <a href="#fr-traitobj-2">↩2</a></p> |
| </li> |
| <li id="footnote-core"> |
| <p>This is the core assumption of <a href="https://github.com/rust-lang/rust/issues/110288">#110288</a> and <a href="https://github.com/rust-lang/rfcs/pull/3417">RFC 3417</a>. <a href="#fr-core-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="region-inference-nll"><a class="header" href="#region-inference-nll">Region inference (NLL)</a></h1> |
| <p>The MIR-based region checking code is located in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/index.html">the <code>rustc_mir::borrow_check</code> |
| module</a>.</p> |
| <p>The MIR-based region analysis consists of two major functions:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.replace_regions_in_mir.html"><code>replace_regions_in_mir</code></a>, invoked first, has two jobs: |
| <ul> |
| <li>First, it finds the set of regions that appear within the |
| signature of the function (e.g., <code>'a</code> in <code>fn foo<'a>(&'a u32) { ... }</code>). These are called the "universal" or "free" regions – in |
| particular, they are the regions that <a href="borrow_check/../appendix/background.html#free-vs-bound">appear free</a> in the |
| function body.</li> |
| <li>Second, it replaces all the regions from the function body with |
| fresh inference variables. This is because (presently) those |
| regions are the results of lexical region inference and hence are |
| not of much interest. The intention is that – eventually – they |
| will be "erased regions" (i.e., no information at all), since we |
| won't be doing lexical region inference at all.</li> |
| </ul> |
| </li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.compute_regions.html"><code>compute_regions</code></a>, invoked second: this is given as argument the |
| results of move analysis. It has the job of computing values for all |
| the inference variables that <code>replace_regions_in_mir</code> introduced. |
| <ul> |
| <li>To do that, it first runs the <a href="borrow_check/./type_check.html">MIR type checker</a>. This is |
| basically a normal type-checker but specialized to MIR, which is |
| much simpler than full Rust, of course. Running the MIR type |
| checker will however create various <a href="borrow_check/./region_inference/constraint_propagation.html">constraints</a> between region |
| variables, indicating their potential values and relationships to |
| one another.</li> |
| <li>After this, we perform <a href="borrow_check/./region_inference/constraint_propagation.html">constraint propagation</a> by creating a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html"><code>RegionInferenceContext</code></a> and invoking its <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.solve"><code>solve</code></a> |
| method.</li> |
| <li>The <a href="https://rust-lang.github.io/rfcs/2094-nll.html">NLL RFC</a> also includes fairly thorough (and hopefully readable) |
| coverage.</li> |
| </ul> |
| </li> |
| </ul> |
| <h2 id="universal-regions"><a class="header" href="#universal-regions">Universal regions</a></h2> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/universal_regions/struct.UniversalRegions.html"><code>UniversalRegions</code></a> type represents a collection of <em>universal</em> regions |
| corresponding to some MIR <code>DefId</code>. It is constructed in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.replace_regions_in_mir.html"><code>replace_regions_in_mir</code></a> when we replace all regions with fresh inference |
| variables. <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/universal_regions/struct.UniversalRegions.html"><code>UniversalRegions</code></a> contains indices for all the free regions in |
| the given MIR along with any relationships that are <em>known</em> to hold between |
| them (e.g. implied bounds, where clauses, etc.).</p> |
| <p>For example, given the MIR for the following function:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn foo<'a>(x: &'a u32) { |
| // ... |
| } |
| <span class="boring">}</span></code></pre></pre> |
| <p>we would create a universal region for <code>'a</code> and one for <code>'static</code>. There may |
| also be some complications for handling closures, but we will ignore those for |
| the moment.</p> |
| <p>TODO: write about <em>how</em> these regions are computed.</p> |
| <p><a id="region-variables"></a></p> |
| <h2 id="region-variables"><a class="header" href="#region-variables">Region variables</a></h2> |
| <p>The value of a region can be thought of as a <strong>set</strong>. This set contains all |
| points in the MIR where the region is valid along with any regions that are |
| outlived by this region (e.g. if <code>'a: 'b</code>, then <code>end('b)</code> is in the set for |
| <code>'a</code>); we call the domain of this set a <code>RegionElement</code>. In the code, the value |
| for all regions is maintained in <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_borrowck/src/region_infer">the <code>rustc_borrowck::region_infer</code> module</a>. |
| For each region we maintain a set storing what elements are present in its value (to make this |
| efficient, we give each kind of element an index, the <code>RegionElementIndex</code>, and |
| use sparse bitsets).</p> |
| <p>The kinds of region elements are as follows:</p> |
| <ul> |
| <li>Each <strong><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Location.html"><code>location</code></a></strong> in the MIR control-flow graph: a location is just |
| the pair of a basic block and an index. This identifies the point |
| <strong>on entry</strong> to the statement with that index (or the terminator, if |
| the index is equal to <code>statements.len()</code>).</li> |
| <li>There is an element <code>end('a)</code> for each universal region <code>'a</code>, |
| corresponding to some portion of the caller's (or caller's caller, |
| etc) control-flow graph.</li> |
| <li>Similarly, there is an element denoted <code>end('static)</code> corresponding |
| to the remainder of program execution after this function returns.</li> |
| <li>There is an element <code>!1</code> for each placeholder region <code>!1</code>. This |
| corresponds (intuitively) to some unknown set of other elements – |
| for details on placeholders, see the section |
| <a href="borrow_check/region_inference/placeholders_and_universes.html">placeholders and universes</a>.</li> |
| </ul> |
| <h2 id="constraints-2"><a class="header" href="#constraints-2">Constraints</a></h2> |
| <p>Before we can infer the value of regions, we need to collect |
| constraints on the regions. The full set of constraints is described |
| in <a href="borrow_check/./region_inference/constraint_propagation.html">the section on constraint propagation</a>, but the two most |
| common sorts of constraints are:</p> |
| <ol> |
| <li>Outlives constraints. These are constraints that one region outlives another |
| (e.g. <code>'a: 'b</code>). Outlives constraints are generated by the <a href="borrow_check/./type_check.html">MIR type |
| checker</a>.</li> |
| <li>Liveness constraints. Each region needs to be live at points where it can be |
| used.</li> |
| </ol> |
| <h2 id="inference-overview"><a class="header" href="#inference-overview">Inference Overview</a></h2> |
| <p>So how do we compute the contents of a region? This process is called <em>region |
| inference</em>. The high-level idea is pretty simple, but there are some details we |
| need to take care of.</p> |
| <p>Here is the high-level idea: we start off each region with the MIR locations we |
| know must be in it from the liveness constraints. From there, we use all of the |
| outlives constraints computed from the type checker to <em>propagate</em> the |
| constraints: for each region <code>'a</code>, if <code>'a: 'b</code>, then we add all elements of |
| <code>'b</code> to <code>'a</code>, including <code>end('b)</code>. This all happens in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.propagate_constraints"><code>propagate_constraints</code></a>.</p> |
| <p>Then, we will check for errors. We first check that type tests are satisfied by |
| calling <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.check_type_tests"><code>check_type_tests</code></a>. This checks constraints like <code>T: 'a</code>. Second, we |
| check that universal regions are not "too big". This is done by calling |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.check_universal_regions"><code>check_universal_regions</code></a>. This checks that for each region <code>'a</code> if <code>'a</code> |
| contains the element <code>end('b)</code>, then we must already know that <code>'a: 'b</code> holds |
| (e.g. from a where clause). If we don't already know this, that is an error... |
| well, almost. There is some special handling for closures that we will discuss |
| later.</p> |
| <h3 id="example-2"><a class="header" href="#example-2">Example</a></h3> |
| <p>Consider the following example:</p> |
| <pre><code class="language-rust ignore">fn foo<'a, 'b>(x: &'a usize) -> &'b usize { |
| x |
| }</code></pre> |
| <p>Clearly, this should not compile because we don't know if <code>'a</code> outlives <code>'b</code> |
| (if it doesn't then the return value could be a dangling reference).</p> |
| <p>Let's back up a bit. We need to introduce some free inference variables (as is |
| done in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.replace_regions_in_mir.html"><code>replace_regions_in_mir</code></a>). This example doesn't use the exact regions |
| produced, but it (hopefully) is enough to get the idea across.</p> |
| <pre><code class="language-rust ignore">fn foo<'a, 'b>(x: &'a /* '#1 */ usize) -> &'b /* '#3 */ usize { |
| x // '#2, location L1 |
| }</code></pre> |
| <p>Some notation: <code>'#1</code>, <code>'#3</code>, and <code>'#2</code> represent the universal regions for the |
| argument, return value, and the expression <code>x</code>, respectively. Additionally, I |
| will call the location of the expression <code>x</code> <code>L1</code>.</p> |
| <p>So now we can use the liveness constraints to get the following starting points:</p> |
| <div class="table-wrapper"><table><thead><tr><th>Region</th><th>Contents</th></tr></thead><tbody> |
| <tr><td>'#1</td><td></td></tr> |
| <tr><td>'#2</td><td><code>L1</code></td></tr> |
| <tr><td>'#3</td><td><code>L1</code></td></tr> |
| </tbody></table> |
| </div> |
| <p>Now we use the outlives constraints to expand each region. Specifically, we |
| know that <code>'#2: '#3</code> ...</p> |
| <div class="table-wrapper"><table><thead><tr><th>Region</th><th>Contents</th></tr></thead><tbody> |
| <tr><td>'#1</td><td><code>L1</code></td></tr> |
| <tr><td>'#2</td><td><code>L1, end('#3) // add contents of '#3 and end('#3)</code></td></tr> |
| <tr><td>'#3</td><td><code>L1</code></td></tr> |
| </tbody></table> |
| </div> |
| <p>... and <code>'#1: '#2</code>, so ...</p> |
| <div class="table-wrapper"><table><thead><tr><th>Region</th><th>Contents</th></tr></thead><tbody> |
| <tr><td>'#1</td><td><code>L1, end('#2), end('#3) // add contents of '#2 and end('#2)</code></td></tr> |
| <tr><td>'#2</td><td><code>L1, end('#3)</code></td></tr> |
| <tr><td>'#3</td><td><code>L1</code></td></tr> |
| </tbody></table> |
| </div> |
| <p>Now, we need to check that no regions were too big (we don't have any type |
| tests to check in this case). Notice that <code>'#1</code> now contains <code>end('#3)</code>, but |
| we have no <code>where</code> clause or implied bound to say that <code>'a: 'b</code>... that's an |
| error!</p> |
| <h3 id="some-details"><a class="header" href="#some-details">Some details</a></h3> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html"><code>RegionInferenceContext</code></a> type contains all of the information needed to |
| do inference, including the universal regions from <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.replace_regions_in_mir.html"><code>replace_regions_in_mir</code></a> and |
| the constraints computed for each region. It is constructed just after we |
| compute the liveness constraints.</p> |
| <p>Here are some of the fields of the struct:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.constraints"><code>constraints</code></a>: contains all the outlives constraints.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.liveness_constraints"><code>liveness_constraints</code></a>: contains all the liveness constraints.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.universal_regions"><code>universal_regions</code></a>: contains the <code>UniversalRegions</code> returned by |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/nll/fn.replace_regions_in_mir.html"><code>replace_regions_in_mir</code></a>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.universal_region_relations"><code>universal_region_relations</code></a>: contains relations known to be true about |
| universal regions. For example, if we have a where clause that <code>'a: 'b</code>, that |
| relation is assumed to be true while borrow checking the implementation (it |
| is checked at the caller), so <code>universal_region_relations</code> would contain <code>'a: 'b</code>.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.type_tests"><code>type_tests</code></a>: contains some constraints on types that we must check after |
| inference (e.g. <code>T: 'a</code>).</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.closure_bounds_mapping"><code>closure_bounds_mapping</code></a>: used for propagating region constraints from |
| closures back out to the creator of the closure.</li> |
| </ul> |
| <p>TODO: should we discuss any of the others fields? What about the SCCs?</p> |
| <p>Ok, now that we have constructed a <code>RegionInferenceContext</code>, we can do |
| inference. This is done by calling the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.solve"><code>solve</code></a> method on the context. This |
| is where we call <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.propagate_constraints"><code>propagate_constraints</code></a> and then check the resulting type |
| tests and universal regions, as discussed above.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="constraint-propagation"><a class="header" href="#constraint-propagation">Constraint propagation</a></h1> |
| <p>The main work of the region inference is <strong>constraint propagation</strong>, |
| which is done in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.propagate_constraints"><code>propagate_constraints</code></a> function. There are |
| three sorts of constraints that are used in NLL, and we'll explain how |
| <code>propagate_constraints</code> works by "layering" those sorts of constraints |
| on one at a time (each of them is fairly independent from the others):</p> |
| <ul> |
| <li>liveness constraints (<code>R live at E</code>), which arise from liveness;</li> |
| <li>outlives constraints (<code>R1: R2</code>), which arise from subtyping;</li> |
| <li><a href="borrow_check/region_inference/./member_constraints.html">member constraints</a> (<code>member R_m of [R_c...]</code>), which arise from impl Trait.</li> |
| </ul> |
| <p>In this chapter, we'll explain the "heart" of constraint propagation, |
| covering both liveness and outlives constraints.</p> |
| <h2 id="notation-and-high-level-concepts"><a class="header" href="#notation-and-high-level-concepts">Notation and high-level concepts</a></h2> |
| <p>Conceptually, region inference is a "fixed-point" computation. It is |
| given some set of constraints <code>{C}</code> and it computes a set of values |
| <code>Values: R -> {E}</code> that maps each region <code>R</code> to a set of elements |
| <code>{E}</code> (see <a href="borrow_check/region_inference/../region_inference.html#region-variables">here</a> for more notes on region elements):</p> |
| <ul> |
| <li>Initially, each region is mapped to an empty set, so <code>Values(R) = {}</code> for all regions <code>R</code>.</li> |
| <li>Next, we process the constraints repeatedly until a fixed-point is reached: |
| <ul> |
| <li>For each constraint C: |
| <ul> |
| <li>Update <code>Values</code> as needed to satisfy the constraint</li> |
| </ul> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p>As a simple example, if we have a liveness constraint <code>R live at E</code>, |
| then we can apply <code>Values(R) = Values(R) union {E}</code> to make the |
| constraint be satisfied. Similarly, if we have an outlives constraints |
| <code>R1: R2</code>, we can apply <code>Values(R1) = Values(R1) union Values(R2)</code>. |
| (Member constraints are more complex and we discuss them <a href="borrow_check/region_inference/./member_constraints.html">in this section</a>.)</p> |
| <p>In practice, however, we are a bit more clever. Instead of applying |
| the constraints in a loop, we can analyze the constraints and figure |
| out the correct order to apply them, so that we only have to apply |
| each constraint once in order to find the final result.</p> |
| <p>Similarly, in the implementation, the <code>Values</code> set is stored in the |
| <code>scc_values</code> field, but they are indexed not by a <em>region</em> but by a |
| <em>strongly connected component</em> (SCC). SCCs are an optimization that |
| avoids a lot of redundant storage and computation. They are explained |
| in the section on outlives constraints.</p> |
| <h2 id="liveness-constraints"><a class="header" href="#liveness-constraints">Liveness constraints</a></h2> |
| <p>A <strong>liveness constraint</strong> arises when some variable whose type |
| includes a region R is live at some <a href="borrow_check/region_inference/../../appendix/glossary.html#point">point</a> P. This simply means that |
| the value of R must include the point P. Liveness constraints are |
| computed by the MIR type checker.</p> |
| <p>A liveness constraint <code>R live at E</code> is satisfied if <code>E</code> is a member of |
| <code>Values(R)</code>. So to "apply" such a constraint to <code>Values</code>, we just have |
| to compute <code>Values(R) = Values(R) union {E}</code>.</p> |
| <p>The liveness values are computed in the type-check and passed to the |
| region inference upon creation in the <code>liveness_constraints</code> argument. |
| These are not represented as individual constraints like <code>R live at E</code> |
| though; instead, we store a (sparse) bitset per region variable (of |
| type <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/values/struct.LivenessValues.html"><code>LivenessValues</code></a>). This way we only need a single bit for each |
| liveness constraint.</p> |
| <p>One thing that is worth mentioning: All lifetime parameters are always |
| considered to be live over the entire function body. This is because |
| they correspond to some portion of the <em>caller's</em> execution, and that |
| execution clearly includes the time spent in this function, since the |
| caller is waiting for us to return.</p> |
| <h2 id="outlives-constraints"><a class="header" href="#outlives-constraints">Outlives constraints</a></h2> |
| <p>An outlives constraint <code>'a: 'b</code> indicates that the value of <code>'a</code> must |
| be a <strong>superset</strong> of the value of <code>'b</code>. That is, an outlives |
| constraint <code>R1: R2</code> is satisfied if <code>Values(R1)</code> is a superset of |
| <code>Values(R2)</code>. So to "apply" such a constraint to <code>Values</code>, we just |
| have to compute <code>Values(R1) = Values(R1) union Values(R2)</code>.</p> |
| <p>One observation that follows from this is that if you have <code>R1: R2</code> |
| and <code>R2: R1</code>, then <code>R1 = R2</code> must be true. Similarly, if you have:</p> |
| <pre><code class="language-txt">R1: R2 |
| R2: R3 |
| R3: R4 |
| R4: R1 |
| </code></pre> |
| <p>then <code>R1 = R2 = R3 = R4</code> follows. We take advantage of this to make things |
| much faster, as described shortly.</p> |
| <p>In the code, the set of outlives constraints is given to the region |
| inference context on creation in a parameter of type |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/constraints/struct.OutlivesConstraintSet.html"><code>OutlivesConstraintSet</code></a>. The constraint set is basically just a list of <code>'a: 'b</code> constraints.</p> |
| <h3 id="the-outlives-constraint-graph-and-sccs"><a class="header" href="#the-outlives-constraint-graph-and-sccs">The outlives constraint graph and SCCs</a></h3> |
| <p>In order to work more efficiently with outlives constraints, they are |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/constraints/struct.OutlivesConstraintSet.html#method.graph">converted into the form of a graph</a>, where the nodes of the |
| graph are region variables (<code>'a</code>, <code>'b</code>) and each constraint <code>'a: 'b</code> |
| induces an edge <code>'a -> 'b</code>. This conversion happens in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.new"><code>RegionInferenceContext::new</code></a> function that creates the inference |
| context.</p> |
| <p>When using a graph representation, we can detect regions that must be equal |
| by looking for cycles. That is, if you have a constraint like</p> |
| <pre><code class="language-txt">'a: 'b |
| 'b: 'c |
| 'c: 'd |
| 'd: 'a |
| </code></pre> |
| <p>then this will correspond to a cycle in the graph containing the |
| elements <code>'a...'d</code>.</p> |
| <p>Therefore, one of the first things that we do in propagating region |
| values is to compute the <strong>strongly connected components</strong> (SCCs) in |
| the constraint graph. The result is stored in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.constraint_sccs"><code>constraint_sccs</code></a> |
| field. You can then easily find the SCC that a region <code>r</code> is a part of |
| by invoking <code>constraint_sccs.scc(r)</code>.</p> |
| <p>Working in terms of SCCs allows us to be more efficient: if we have a |
| set of regions <code>'a...'d</code> that are part of a single SCC, we don't have |
| to compute/store their values separately. We can just store one value |
| <strong>for the SCC</strong>, since they must all be equal.</p> |
| <p>If you look over the region inference code, you will see that a number |
| of fields are defined in terms of SCCs. For example, the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#structfield.scc_values"><code>scc_values</code></a> field stores the values of each SCC. To get the value |
| of a specific region <code>'a</code> then, we first figure out the SCC that the |
| region is a part of, and then find the value of that SCC.</p> |
| <p>When we compute SCCs, we not only figure out which regions are a |
| member of each SCC, we also figure out the edges between them. So for example |
| consider this set of outlives constraints:</p> |
| <pre><code class="language-txt">'a: 'b |
| 'b: 'a |
| |
| 'a: 'c |
| |
| 'c: 'd |
| 'd: 'c |
| </code></pre> |
| <p>Here we have two SCCs: S0 contains <code>'a</code> and <code>'b</code>, and S1 contains <code>'c</code> |
| and <code>'d</code>. But these SCCs are not independent: because <code>'a: 'c</code>, that |
| means that <code>S0: S1</code> as well. That is -- the value of <code>S0</code> must be a |
| superset of the value of <code>S1</code>. One crucial thing is that this graph of |
| SCCs is always a DAG -- that is, it never has cycles. This is because |
| all the cycles have been removed to form the SCCs themselves.</p> |
| <h3 id="applying-liveness-constraints-to-sccs"><a class="header" href="#applying-liveness-constraints-to-sccs">Applying liveness constraints to SCCs</a></h3> |
| <p>The liveness constraints that come in from the type-checker are |
| expressed in terms of regions -- that is, we have a map like |
| <code>Liveness: R -> {E}</code>. But we want our final result to be expressed |
| in terms of SCCs -- we can integrate these liveness constraints very |
| easily just by taking the union:</p> |
| <pre><code class="language-txt">for each region R: |
| let S be the SCC that contains R |
| Values(S) = Values(S) union Liveness(R) |
| </code></pre> |
| <p>In the region inferencer, this step is done in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.new"><code>RegionInferenceContext::new</code></a>.</p> |
| <h3 id="applying-outlives-constraints"><a class="header" href="#applying-outlives-constraints">Applying outlives constraints</a></h3> |
| <p>Once we have computed the DAG of SCCs, we use that to structure out |
| entire computation. If we have an edge <code>S1 -> S2</code> between two SCCs, |
| that means that <code>Values(S1) >= Values(S2)</code> must hold. So, to compute |
| the value of <code>S1</code>, we first compute the values of each successor <code>S2</code>. |
| Then we simply union all of those values together. To use a |
| quasi-iterator-like notation:</p> |
| <pre><code class="language-txt">Values(S1) = |
| s1.successors() |
| .map(|s2| Values(s2)) |
| .union() |
| </code></pre> |
| <p>In the code, this work starts in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.propagate_constraints"><code>propagate_constraints</code></a> |
| function, which iterates over all the SCCs. For each SCC <code>S1</code>, we |
| compute its value by first computing the value of its |
| successors. Since SCCs form a DAG, we don't have to be concerned about |
| cycles, though we do need to keep a set around to track whether we |
| have already processed a given SCC or not. For each successor <code>S2</code>, once |
| we have computed <code>S2</code>'s value, we can union those elements into the |
| value for <code>S1</code>. (Although we have to be careful in this process to |
| properly handle <a href="borrow_check/region_inference/./placeholders_and_universes.html">higher-ranked |
| placeholders</a>. Note that the value |
| for <code>S1</code> already contains the liveness constraints, since they were |
| added in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.new"><code>RegionInferenceContext::new</code></a>.</p> |
| <p>Once that process is done, we now have the "minimal value" for <code>S1</code>, |
| taking into account all of the liveness and outlives |
| constraints. However, in order to complete the process, we must also |
| consider <a href="borrow_check/region_inference/./member_constraints.html">member constraints</a>, which are described in <a href="borrow_check/region_inference/./member_constraints.html">a later |
| section</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="universal-regions-1"><a class="header" href="#universal-regions-1">Universal regions</a></h1> |
| <p>"Universal regions" is the name that the code uses to refer to "named |
| lifetimes" -- e.g., lifetime parameters and <code>'static</code>. The name |
| derives from the fact that such lifetimes are "universally quantified" |
| (i.e., we must make sure the code is true for all values of those |
| lifetimes). It is worth spending a bit of discussing how lifetime |
| parameters are handled during region inference. Consider this example:</p> |
| <pre><code class="language-rust ignore">fn foo<'a, 'b>(x: &'a u32, y: &'b u32) -> &'b u32 { |
| x |
| }</code></pre> |
| <p>This example is intended not to compile, because we are returning <code>x</code>, |
| which has type <code>&'a u32</code>, but our signature promises that we will |
| return a <code>&'b u32</code> value. But how are lifetimes like <code>'a</code> and <code>'b</code> |
| integrated into region inference, and how this error wind up being |
| detected?</p> |
| <h2 id="universal-regions-and-their-relationships-to-one-another"><a class="header" href="#universal-regions-and-their-relationships-to-one-another">Universal regions and their relationships to one another</a></h2> |
| <p>Early on in region inference, one of the first things we do is to |
| construct a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/universal_regions/struct.UniversalRegions.html"><code>UniversalRegions</code></a> struct. This struct tracks the |
| various universal regions in scope on a particular function. We also |
| create a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/free_region_relations/struct.UniversalRegionRelations.html"><code>UniversalRegionRelations</code></a> struct, which tracks their |
| relationships to one another. So if you have e.g. <code>where 'a: 'b</code>, then |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/free_region_relations/struct.UniversalRegionRelations.html"><code>UniversalRegionRelations</code></a> struct would track that <code>'a: 'b</code> is |
| known to hold (which could be tested with the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/free_region_relations/struct.UniversalRegionRelations.html#method.outlives"><code>outlives</code></a> function).</p> |
| <h2 id="everything-is-a-region-variable"><a class="header" href="#everything-is-a-region-variable">Everything is a region variable</a></h2> |
| <p>One important aspect of how NLL region inference works is that <strong>all |
| lifetimes</strong> are represented as numbered variables. This means that the |
| only variant of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/region_kind/enum.RegionKind.html"><code>region_kind::RegionKind</code></a> that we use is the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/region_kind/enum.RegionKind.html#variant.ReVar"><code>ReVar</code></a> |
| variant. These region variables are broken into two major categories, |
| based on their index:</p> |
| <ul> |
| <li>0..N: universal regions -- the ones we are discussing here. In this |
| case, the code must be correct with respect to any value of those |
| variables that meets the declared relationships.</li> |
| <li>N..M: existential regions -- inference variables where the region |
| inferencer is tasked with finding <em>some</em> suitable value.</li> |
| </ul> |
| <p>In fact, the universal regions can be further subdivided based on |
| where they were brought into scope (see the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/universal_regions/enum.RegionClassification.html#variant.Local"><code>RegionClassification</code></a> |
| type). These subdivisions are not important for the topics discussed |
| here, but become important when we consider <a href="borrow_check/region_inference/./closure_constraints.html">closure constraint |
| propagation</a>, so we discuss them there.</p> |
| <h2 id="universal-lifetimes-as-the-elements-of-a-regions-value"><a class="header" href="#universal-lifetimes-as-the-elements-of-a-regions-value">Universal lifetimes as the elements of a region's value</a></h2> |
| <p>As noted previously, the value that we infer for each region is a set |
| <code>{E}</code>. The elements of this set can be points in the control-flow |
| graph, but they can also be an element <code>end('a)</code> corresponding to each |
| universal lifetime <code>'a</code>. If the value for some region <code>R0</code> includes |
| <code>end('a</code>), then this implies that <code>R0</code> must extend until the end of <code>'a</code> |
| in the caller.</p> |
| <h2 id="the-value-of-a-universal-region"><a class="header" href="#the-value-of-a-universal-region">The "value" of a universal region</a></h2> |
| <p>During region inference, we compute a value for each universal region |
| in the same way as we compute values for other regions. This value |
| represents, effectively, the <strong>lower bound</strong> on that universal region |
| -- the things that it must outlive. We now describe how we use this |
| value to check for errors.</p> |
| <h2 id="liveness-and-universal-regions"><a class="header" href="#liveness-and-universal-regions">Liveness and universal regions</a></h2> |
| <p>All universal regions have an initial liveness constraint that |
| includes the entire function body. This is because lifetime parameters |
| are defined in the caller and must include the entirety of the |
| function call that invokes this particular function. In addition, each |
| universal region <code>'a</code> includes itself (that is, <code>end('a)</code>) in its |
| liveness constraint (i.e., <code>'a</code> must extend until the end of |
| itself). In the code, these liveness constraints are setup in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.init_free_and_bound_regions"><code>init_free_and_bound_regions</code></a>.</p> |
| <h2 id="propagating-outlives-constraints-for-universal-regions"><a class="header" href="#propagating-outlives-constraints-for-universal-regions">Propagating outlives constraints for universal regions</a></h2> |
| <p>So, consider the first example of this section:</p> |
| <pre><code class="language-rust ignore">fn foo<'a, 'b>(x: &'a u32, y: &'b u32) -> &'b u32 { |
| x |
| }</code></pre> |
| <p>Here, returning <code>x</code> requires that <code>&'a u32 <: &'b u32</code>, which gives |
| rise to an outlives constraint <code>'a: 'b</code>. Combined with our default liveness |
| constraints we get:</p> |
| <pre><code class="language-txt">'a live at {B, end('a)} // B represents the "function body" |
| 'b live at {B, end('b)} |
| 'a: 'b |
| </code></pre> |
| <p>When we process the <code>'a: 'b</code> constraint, therefore, we will add |
| <code>end('b)</code> into the value for <code>'a</code>, resulting in a final value of <code>{B, end('a), end('b)}</code>.</p> |
| <h2 id="detecting-errors"><a class="header" href="#detecting-errors">Detecting errors</a></h2> |
| <p>Once we have finished constraint propagation, we then enforce a |
| constraint that if some universal region <code>'a</code> includes an element |
| <code>end('b)</code>, then <code>'a: 'b</code> must be declared in the function's bounds. If |
| not, as in our example, that is an error. This check is done in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/region_infer/struct.RegionInferenceContext.html#method.check_universal_regions"><code>check_universal_regions</code></a> function, which simply iterates over all |
| universal regions, inspects their final value, and tests against the |
| declared <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/free_region_relations/struct.UniversalRegionRelations.html"><code>UniversalRegionRelations</code></a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="member-constraints-1"><a class="header" href="#member-constraints-1">Member constraints</a></h1> |
| <p>A member constraint <code>'m member of ['c_1..'c_N]</code> expresses that the |
| region <code>'m</code> must be <em>equal</em> to some <strong>choice regions</strong> <code>'c_i</code> (for |
| some <code>i</code>). These constraints cannot be expressed by users, but they |
| arise from <code>impl Trait</code> due to its lifetime capture rules. Consider a |
| function such as the following:</p> |
| <pre><code class="language-rust ignore">fn make(a: &'a u32, b: &'b u32) -> impl Trait<'a, 'b> { .. }</code></pre> |
| <p>Here, the true return type (often called the "hidden type") is only |
| permitted to capture the lifetimes <code>'a</code> or <code>'b</code>. You can kind of see |
| this more clearly by desugaring that <code>impl Trait</code> return type into its |
| more explicit form:</p> |
| <pre><code class="language-rust ignore">type MakeReturn<'x, 'y> = impl Trait<'x, 'y>; |
| fn make(a: &'a u32, b: &'b u32) -> MakeReturn<'a, 'b> { .. }</code></pre> |
| <p>Here, the idea is that the hidden type must be some type that could |
| have been written in place of the <code>impl Trait<'x, 'y></code> -- but clearly |
| such a type can only reference the regions <code>'x</code> or <code>'y</code> (or |
| <code>'static</code>!), as those are the only names in scope. This limitation is |
| then translated into a restriction to only access <code>'a</code> or <code>'b</code> because |
| we are returning <code>MakeReturn<'a, 'b></code>, where <code>'x</code> and <code>'y</code> have been |
| replaced with <code>'a</code> and <code>'b</code> respectively.</p> |
| <h2 id="detailed-example"><a class="header" href="#detailed-example">Detailed example</a></h2> |
| <p>To help us explain member constraints in more detail, let's spell out |
| the <code>make</code> example in a bit more detail. First off, let's assume that |
| you have some dummy trait:</p> |
| <pre><code class="language-rust ignore">trait Trait<'a, 'b> { } |
| impl<T> Trait<'_, '_> for T { }</code></pre> |
| <p>and this is the <code>make</code> function (in desugared form):</p> |
| <pre><code class="language-rust ignore">type MakeReturn<'x, 'y> = impl Trait<'x, 'y>; |
| fn make(a: &'a u32, b: &'b u32) -> MakeReturn<'a, 'b> { |
| (a, b) |
| }</code></pre> |
| <p>What happens in this case is that the return type will be <code>(&'0 u32, &'1 u32)</code>, |
| where <code>'0</code> and <code>'1</code> are fresh region variables. We will have the following |
| region constraints:</p> |
| <pre><code class="language-txt">'0 live at {L} |
| '1 live at {L} |
| 'a: '0 |
| 'b: '1 |
| '0 member of ['a, 'b, 'static] |
| '1 member of ['a, 'b, 'static] |
| </code></pre> |
| <p>Here the "liveness set" <code>{L}</code> corresponds to that subset of the body |
| where <code>'0</code> and <code>'1</code> are live -- basically the point from where the |
| return tuple is constructed to where it is returned (in fact, <code>'0</code> and |
| <code>'1</code> might have slightly different liveness sets, but that's not very |
| interesting to the point we are illustrating here).</p> |
| <p>The <code>'a: '0</code> and <code>'b: '1</code> constraints arise from subtyping. When we |
| construct the <code>(a, b)</code> value, it will be assigned type <code>(&'0 u32, &'1 u32)</code> -- the region variables reflect that the lifetimes of these |
| references could be made smaller. For this value to be created from |
| <code>a</code> and <code>b</code>, however, we do require that:</p> |
| <pre><code class="language-txt">(&'a u32, &'b u32) <: (&'0 u32, &'1 u32) |
| </code></pre> |
| <p>which means in turn that <code>&'a u32 <: &'0 u32</code> and hence that <code>'a: '0</code> |
| (and similarly that <code>&'b u32 <: &'1 u32</code>, <code>'b: '1</code>).</p> |
| <p>Note that if we ignore member constraints, the value of <code>'0</code> would be |
| inferred to some subset of the function body (from the liveness |
| constraints, which we did not write explicitly). It would never become |
| <code>'a</code>, because there is no need for it too -- we have a constraint that |
| <code>'a: '0</code>, but that just puts a "cap" on how <em>large</em> <code>'0</code> can grow to |
| become. Since we compute the <em>minimal</em> value that we can, we are happy |
| to leave <code>'0</code> as being just equal to the liveness set. This is where |
| member constraints come in.</p> |
| <h2 id="choices-are-always-lifetime-parameters"><a class="header" href="#choices-are-always-lifetime-parameters">Choices are always lifetime parameters</a></h2> |
| <p>At present, the "choice" regions from a member constraint are always lifetime |
| parameters from the current function. As of <!-- date-check --> October 2021, |
| this falls out from the placement of impl Trait, though in the future it may not |
| be the case. We take some advantage of this fact, as it simplifies the current |
| code. In particular, we don't have to consider a case like <code>'0 member of ['1, 'static]</code>, in which the value of both <code>'0</code> and <code>'1</code> are being inferred and hence |
| changing. See <a href="https://github.com/rust-lang/rust/issues/61773">rust-lang/rust#61773</a> for more information.</p> |
| <h2 id="applying-member-constraints"><a class="header" href="#applying-member-constraints">Applying member constraints</a></h2> |
| <p>Member constraints are a bit more complex than other forms of |
| constraints. This is because they have a "or" quality to them -- that |
| is, they describe multiple choices that we must select from. E.g., in |
| our example constraint <code>'0 member of ['a, 'b, 'static]</code>, it might be |
| that <code>'0</code> is equal to <code>'a</code>, <code>'b</code>, <em>or</em> <code>'static</code>. How can we pick the |
| correct one? What we currently do is to look for a <em>minimal choice</em> |
| -- if we find one, then we will grow <code>'0</code> to be equal to that minimal |
| choice. To find that minimal choice, we take two factors into |
| consideration: lower and upper bounds.</p> |
| <h3 id="lower-bounds"><a class="header" href="#lower-bounds">Lower bounds</a></h3> |
| <p>The <em>lower bounds</em> are those lifetimes that <code>'0</code> <em>must outlive</em> -- |
| i.e., that <code>'0</code> must be larger than. In fact, when it comes time to |
| apply member constraints, we've already <em>computed</em> the lower bounds of |
| <code>'0</code> because we computed its minimal value (or at least, the lower |
| bounds considering everything but member constraints).</p> |
| <p>Let <code>LB</code> be the current value of <code>'0</code>. We know then that <code>'0: LB</code> must |
| hold, whatever the final value of <code>'0</code> is. Therefore, we can rule out |
| any choice <code>'choice</code> where <code>'choice: LB</code> does not hold.</p> |
| <p>Unfortunately, in our example, this is not very helpful. The lower |
| bound for <code>'0</code> will just be the liveness set <code>{L}</code>, and we know that |
| all the lifetime parameters outlive that set. So we are left with the |
| same set of choices here. (But in other examples, particularly those |
| with different variance, lower bound constraints may be relevant.)</p> |
| <h3 id="upper-bounds"><a class="header" href="#upper-bounds">Upper bounds</a></h3> |
| <p>The <em>upper bounds</em> are those lifetimes that <em>must outlive</em> <code>'0</code> -- |
| i.e., that <code>'0</code> must be <em>smaller</em> than. In our example, this would be |
| <code>'a</code>, because we have the constraint that <code>'a: '0</code>. In more complex |
| examples, the chain may be more indirect.</p> |
| <p>We can use upper bounds to rule out members in a very similar way to |
| lower bounds. If UB is some upper bound, then we know that <code>UB: '0</code> must hold, so we can rule out any choice <code>'choice</code> where <code>UB: 'choice</code> does not hold.</p> |
| <p>In our example, we would be able to reduce our choice set from <code>['a, 'b, 'static]</code> to just <code>['a]</code>. This is because <code>'0</code> has an upper bound |
| of <code>'a</code>, and neither <code>'a: 'b</code> nor <code>'a: 'static</code> is known to hold.</p> |
| <p>(For notes on how we collect upper bounds in the implementation, see |
| <a href="borrow_check/region_inference/member_constraints.html#collecting">the section below</a>.)</p> |
| <h3 id="minimal-choice"><a class="header" href="#minimal-choice">Minimal choice</a></h3> |
| <p>After applying lower and upper bounds, we can still sometimes have |
| multiple possibilities. For example, imagine a variant of our example |
| using types with the opposite variance. In that case, we would have |
| the constraint <code>'0: 'a</code> instead of <code>'a: '0</code>. Hence the current value |
| of <code>'0</code> would be <code>{L, 'a}</code>. Using this as a lower bound, we would be |
| able to narrow down the member choices to <code>['a, 'static]</code> because <code>'b: 'a</code> is not known to hold (but <code>'a: 'a</code> and <code>'static: 'a</code> do hold). We |
| would not have any upper bounds, so that would be our final set of choices.</p> |
| <p>In that case, we apply the <strong>minimal choice</strong> rule -- basically, if |
| one of our choices if smaller than the others, we can use that. In |
| this case, we would opt for <code>'a</code> (and not <code>'static</code>).</p> |
| <p>This choice is consistent with the general 'flow' of region |
| propagation, which always aims to compute a minimal value for the |
| region being inferred. However, it is somewhat arbitrary.</p> |
| <p><a id="collecting"></a></p> |
| <h3 id="collecting-upper-bounds-in-the-implementation"><a class="header" href="#collecting-upper-bounds-in-the-implementation">Collecting upper bounds in the implementation</a></h3> |
| <p>In practice, computing upper bounds is a bit inconvenient, because our |
| data structures are setup for the opposite. What we do is to compute |
| the <strong>reverse SCC graph</strong> (we do this lazily and cache the result) -- |
| that is, a graph where <code>'a: 'b</code> induces an edge <code>SCC('b) -> SCC('a)</code>. Like the normal SCC graph, this is a DAG. We can then do a |
| depth-first search starting from <code>SCC('0)</code> in this graph. This will |
| take us to all the SCCs that must outlive <code>'0</code>.</p> |
| <p>One wrinkle is that, as we walk the "upper bound" SCCs, their values |
| will not yet have been fully computed. However, we <strong>have</strong> already |
| applied their liveness constraints, so we have some information about |
| their value. In particular, for any regions representing lifetime |
| parameters, their value will contain themselves (i.e., the initial |
| value for <code>'a</code> includes <code>'a</code> and the value for <code>'b</code> contains <code>'b</code>). So |
| we can collect all of the lifetime parameters that are reachable, |
| which is precisely what we are interested in.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="placeholders-and-universes"><a class="header" href="#placeholders-and-universes">Placeholders and universes</a></h1> |
| <p>From time to time we have to reason about regions that we can't |
| concretely know. For example, consider this program:</p> |
| <pre><code class="language-rust ignore">// A function that needs a static reference |
| fn foo(x: &'static u32) { } |
| |
| fn bar(f: for<'a> fn(&'a u32)) { |
| // ^^^^^^^^^^^^^^^^^^^ a function that can accept **any** reference |
| let x = 22; |
| f(&x); |
| } |
| |
| fn main() { |
| bar(foo); |
| }</code></pre> |
| <p>This program ought not to type-check: <code>foo</code> needs a static reference |
| for its argument, and <code>bar</code> wants to be given a function that |
| accepts <strong>any</strong> reference (so it can call it with something on its |
| stack, for example). But <em>how</em> do we reject it and <em>why</em>?</p> |
| <h2 id="subtyping-and-placeholders"><a class="header" href="#subtyping-and-placeholders">Subtyping and Placeholders</a></h2> |
| <p>When we type-check <code>main</code>, and in particular the call <code>bar(foo)</code>, we |
| are going to wind up with a subtyping relationship like this one:</p> |
| <pre><code class="language-text">fn(&'static u32) <: for<'a> fn(&'a u32) |
| ---------------- ------------------- |
| the type of `foo` the type `bar` expects |
| </code></pre> |
| <p>We handle this sort of subtyping by taking the variables that are |
| bound in the supertype and replacing them with |
| <a href="borrow_check/region_inference/../../appendix/background.html#quantified">universally quantified</a> |
| representatives, denoted like <code>!1</code> here. We call these regions "placeholder |
| regions" – they represent, basically, "some unknown region".</p> |
| <p>Once we've done that replacement, we have the following relation:</p> |
| <pre><code class="language-text">fn(&'static u32) <: fn(&'!1 u32) |
| </code></pre> |
| <p>The key idea here is that this unknown region <code>'!1</code> is not related to |
| any other regions. So if we can prove that the subtyping relationship |
| is true for <code>'!1</code>, then it ought to be true for any region, which is |
| what we wanted.</p> |
| <p>So let's work through what happens next. To check if two functions are |
| subtypes, we check if their arguments have the desired relationship |
| (fn arguments are <a href="borrow_check/region_inference/../../appendix/background.html#variance">contravariant</a>, so |
| we swap the left and right here):</p> |
| <pre><code class="language-text">&'!1 u32 <: &'static u32 |
| </code></pre> |
| <p>According to the basic subtyping rules for a reference, this will be |
| true if <code>'!1: 'static</code>. That is – if "some unknown region <code>!1</code>" outlives <code>'static</code>. |
| Now, this <em>might</em> be true – after all, <code>'!1</code> could be <code>'static</code> – |
| but we don't <em>know</em> that it's true. So this should yield up an error (eventually).</p> |
| <h2 id="what-is-a-universe"><a class="header" href="#what-is-a-universe">What is a universe?</a></h2> |
| <p>In the previous section, we introduced the idea of a placeholder |
| region, and we denoted it <code>!1</code>. We call this number <code>1</code> the <strong>universe |
| index</strong>. The idea of a "universe" is that it is a set of names that |
| are in scope within some type or at some point. Universes are formed |
| into a tree, where each child extends its parents with some new names. |
| So the <strong>root universe</strong> conceptually contains global names, such as |
| the lifetime <code>'static</code> or the type <code>i32</code>. In the compiler, we also |
| put generic type parameters into this root universe (in this sense, |
| there is not just one root universe, but one per item). So consider |
| this function <code>bar</code>:</p> |
| <pre><code class="language-rust ignore">struct Foo { } |
| |
| fn bar<'a, T>(t: &'a T) { |
| ... |
| }</code></pre> |
| <p>Here, the root universe would consist of the lifetimes <code>'static</code> and |
| <code>'a</code>. In fact, although we're focused on lifetimes here, we can apply |
| the same concept to types, in which case the types <code>Foo</code> and <code>T</code> would |
| be in the root universe (along with other global types, like <code>i32</code>). |
| Basically, the root universe contains all the names that |
| <a href="borrow_check/region_inference/../../appendix/background.html#free-vs-bound">appear free</a> in the body of <code>bar</code>.</p> |
| <p>Now let's extend <code>bar</code> a bit by adding a variable <code>x</code>:</p> |
| <pre><code class="language-rust ignore">fn bar<'a, T>(t: &'a T) { |
| let x: for<'b> fn(&'b u32) = ...; |
| }</code></pre> |
| <p>Here, the name <code>'b</code> is not part of the root universe. Instead, when we |
| "enter" into this <code>for<'b></code> (e.g., by replacing it with a placeholder), we will create |
| a child universe of the root, let's call it U1:</p> |
| <pre><code class="language-text">U0 (root universe) |
| │ |
| └─ U1 (child universe) |
| </code></pre> |
| <p>The idea is that this child universe U1 extends the root universe U0 |
| with a new name, which we are identifying by its universe number: |
| <code>!1</code>.</p> |
| <p>Now let's extend <code>bar</code> a bit by adding one more variable, <code>y</code>:</p> |
| <pre><code class="language-rust ignore">fn bar<'a, T>(t: &'a T) { |
| let x: for<'b> fn(&'b u32) = ...; |
| let y: for<'c> fn(&'c u32) = ...; |
| }</code></pre> |
| <p>When we enter <em>this</em> type, we will again create a new universe, which |
| we'll call <code>U2</code>. Its parent will be the root universe, and U1 will be |
| its sibling:</p> |
| <pre><code class="language-text">U0 (root universe) |
| │ |
| ├─ U1 (child universe) |
| │ |
| └─ U2 (child universe) |
| </code></pre> |
| <p>This implies that, while in U2, we can name things from U0 or U2, but |
| not U1.</p> |
| <p><strong>Giving existential variables a universe.</strong> Now that we have this |
| notion of universes, we can use it to extend our type-checker and |
| things to prevent illegal names from leaking out. The idea is that we |
| give each inference (existential) variable – whether it be a type or |
| a lifetime – a universe. That variable's value can then only |
| reference names visible from that universe. So for example if a |
| lifetime variable is created in U0, then it cannot be assigned a value |
| of <code>!1</code> or <code>!2</code>, because those names are not visible from the universe |
| U0.</p> |
| <p><strong>Representing universes with just a counter.</strong> You might be surprised |
| to see that the compiler doesn't keep track of a full tree of |
| universes. Instead, it just keeps a counter – and, to determine if |
| one universe can see another one, it just checks if the index is |
| greater. For example, U2 can see U0 because 2 >= 0. But U0 cannot see |
| U2, because 0 >= 2 is false.</p> |
| <p>How can we get away with this? Doesn't this mean that we would allow |
| U2 to also see U1? The answer is that, yes, we would, <strong>if that |
| question ever arose</strong>. But because of the structure of our type |
| checker etc, there is no way for that to happen. In order for |
| something happening in the universe U1 to "communicate" with something |
| happening in U2, they would have to have a shared inference variable X |
| in common. And because everything in U1 is scoped to just U1 and its |
| children, that inference variable X would have to be in U0. And since |
| X is in U0, it cannot name anything from U1 (or U2). This is perhaps easiest |
| to see by using a kind of generic "logic" example:</p> |
| <pre><code class="language-text">exists<X> { |
| forall<Y> { ... /* Y is in U1 ... */ } |
| forall<Z> { ... /* Z is in U2 ... */ } |
| } |
| </code></pre> |
| <p>Here, the only way for the two foralls to interact would be through X, |
| but neither Y nor Z are in scope when X is declared, so its value |
| cannot reference either of them.</p> |
| <h2 id="universes-and-placeholder-region-elements"><a class="header" href="#universes-and-placeholder-region-elements">Universes and placeholder region elements</a></h2> |
| <p>But where does that error come from? The way it happens is like this. |
| When we are constructing the region inference context, we can tell |
| from the type inference context how many placeholder variables exist |
| (the <code>InferCtxt</code> has an internal counter). For each of those, we |
| create a corresponding universal region variable <code>!n</code> and a "region |
| element" <code>placeholder(n)</code>. This corresponds to "some unknown set of other |
| elements". The value of <code>!n</code> is <code>{placeholder(n)}</code>.</p> |
| <p>At the same time, we also give each existential variable a |
| <strong>universe</strong> (also taken from the <code>InferCtxt</code>). This universe |
| determines which placeholder elements may appear in its value: For |
| example, a variable in universe U3 may name <code>placeholder(1)</code>, <code>placeholder(2)</code>, and |
| <code>placeholder(3)</code>, but not <code>placeholder(4)</code>. Note that the universe of an inference |
| variable controls what region elements <strong>can</strong> appear in its value; it |
| does not say region elements <strong>will</strong> appear.</p> |
| <h2 id="placeholders-and-outlives-constraints"><a class="header" href="#placeholders-and-outlives-constraints">Placeholders and outlives constraints</a></h2> |
| <p>In the region inference engine, outlives constraints have the form:</p> |
| <pre><code class="language-text">V1: V2 @ P |
| </code></pre> |
| <p>where <code>V1</code> and <code>V2</code> are region indices, and hence map to some region |
| variable (which may be universally or existentially quantified). The |
| <code>P</code> here is a "point" in the control-flow graph; it's not important |
| for this section. This variable will have a universe, so let's call |
| those universes <code>U(V1)</code> and <code>U(V2)</code> respectively. (Actually, the only |
| one we are going to care about is <code>U(V1)</code>.)</p> |
| <p>When we encounter this constraint, the ordinary procedure is to start |
| a DFS from <code>P</code>. We keep walking so long as the nodes we are walking |
| are present in <code>value(V2)</code> and we add those nodes to <code>value(V1)</code>. If |
| we reach a return point, we add in any <code>end(X)</code> elements. That part |
| remains unchanged.</p> |
| <p>But then <em>after that</em> we want to iterate over the placeholder <code>placeholder(x)</code> |
| elements in V2 (each of those must be visible to <code>U(V2)</code>, but we |
| should be able to just assume that is true, we don't have to check |
| it). We have to ensure that <code>value(V1)</code> outlives each of those |
| placeholder elements.</p> |
| <p>Now there are two ways that could happen. First, if <code>U(V1)</code> can see |
| the universe <code>x</code> (i.e., <code>x <= U(V1)</code>), then we can just add <code>placeholder(x)</code> |
| to <code>value(V1)</code> and be done. But if not, then we have to approximate: |
| we may not know what set of elements <code>placeholder(x)</code> represents, but we |
| should be able to compute some sort of <strong>upper bound</strong> B for it – |
| some region B that outlives <code>placeholder(x)</code>. For now, we'll just use |
| <code>'static</code> for that (since it outlives everything) – in the future, we |
| can sometimes be smarter here (and in fact we have code for doing this |
| already in other contexts). Moreover, since <code>'static</code> is in the root |
| universe U0, we know that all variables can see it – so basically if |
| we find that <code>value(V2)</code> contains <code>placeholder(x)</code> for some universe <code>x</code> |
| that <code>V1</code> can't see, then we force <code>V1</code> to <code>'static</code>.</p> |
| <h2 id="extending-the-universal-regions-check"><a class="header" href="#extending-the-universal-regions-check">Extending the "universal regions" check</a></h2> |
| <p>After all constraints have been propagated, the NLL region inference |
| has one final check, where it goes over the values that wound up being |
| computed for each universal region and checks that they did not get |
| 'too large'. In our case, we will go through each placeholder region |
| and check that it contains <em>only</em> the <code>placeholder(u)</code> element it is known to |
| outlive. (Later, we might be able to know that there are relationships |
| between two placeholder regions and take those into account, as we do |
| for universal regions from the fn signature.)</p> |
| <p>Put another way, the "universal regions" check can be considered to be |
| checking constraints like:</p> |
| <pre><code class="language-text">{placeholder(1)}: V1 |
| </code></pre> |
| <p>where <code>{placeholder(1)}</code> is like a constant set, and V1 is the variable we |
| made to represent the <code>!1</code> region.</p> |
| <h2 id="back-to-our-example"><a class="header" href="#back-to-our-example">Back to our example</a></h2> |
| <p>OK, so far so good. Now let's walk through what would happen with our |
| first example:</p> |
| <pre><code class="language-text">fn(&'static u32) <: fn(&'!1 u32) @ P // this point P is not imp't here |
| </code></pre> |
| <p>The region inference engine will create a region element domain like this:</p> |
| <pre><code class="language-text">{ CFG; end('static); placeholder(1) } |
| --- ------------ ------- from the universe `!1` |
| | 'static is always in scope |
| all points in the CFG; not especially relevant here |
| </code></pre> |
| <p>It will always create two universal variables, one representing |
| <code>'static</code> and one representing <code>'!1</code>. Let's call them Vs and V1. They |
| will have initial values like so:</p> |
| <pre><code class="language-text">Vs = { CFG; end('static) } // it is in U0, so can't name anything else |
| V1 = { placeholder(1) } |
| </code></pre> |
| <p>From the subtyping constraint above, we would have an outlives constraint like</p> |
| <pre><code class="language-text">'!1: 'static @ P |
| </code></pre> |
| <p>To process this, we would grow the value of V1 to include all of Vs:</p> |
| <pre><code class="language-text">Vs = { CFG; end('static) } |
| V1 = { CFG; end('static), placeholder(1) } |
| </code></pre> |
| <p>At that point, constraint propagation is complete, because all the |
| outlives relationships are satisfied. Then we would go to the "check |
| universal regions" portion of the code, which would test that no |
| universal region grew too large.</p> |
| <p>In this case, <code>V1</code> <em>did</em> grow too large – it is not known to outlive |
| <code>end('static)</code>, nor any of the CFG – so we would report an error.</p> |
| <h2 id="another-example"><a class="header" href="#another-example">Another example</a></h2> |
| <p>What about this subtyping relationship?</p> |
| <pre><code class="language-text">for<'a> fn(&'a u32, &'a u32) |
| <: |
| for<'b, 'c> fn(&'b u32, &'c u32) |
| </code></pre> |
| <p>Here we would replace the bound region in the supertype with a placeholder, as before, yielding:</p> |
| <pre><code class="language-text">for<'a> fn(&'a u32, &'a u32) |
| <: |
| fn(&'!1 u32, &'!2 u32) |
| </code></pre> |
| <p>then we instantiate the variable on the left-hand side with an |
| existential in universe U2, yielding the following (<code>?n</code> is a notation |
| for an existential variable):</p> |
| <pre><code class="language-text">fn(&'?3 u32, &'?3 u32) |
| <: |
| fn(&'!1 u32, &'!2 u32) |
| </code></pre> |
| <p>Then we break this down further:</p> |
| <pre><code class="language-text">&'!1 u32 <: &'?3 u32 |
| &'!2 u32 <: &'?3 u32 |
| </code></pre> |
| <p>and even further, yield up our region constraints:</p> |
| <pre><code class="language-text">'!1: '?3 |
| '!2: '?3 |
| </code></pre> |
| <p>Note that, in this case, both <code>'!1</code> and <code>'!2</code> have to outlive the |
| variable <code>'?3</code>, but the variable <code>'?3</code> is not forced to outlive |
| anything else. Therefore, it simply starts and ends as the empty set |
| of elements, and hence the type-check succeeds here.</p> |
| <p>(This should surprise you a little. It surprised me when I first realized it. |
| We are saying that if we are a fn that <strong>needs both of its arguments to have |
| the same region</strong>, we can accept being called with <strong>arguments with two |
| distinct regions</strong>. That seems intuitively unsound. But in fact, it's fine, as |
| I tried to explain in <a href="https://github.com/rust-lang/rust/issues/32330#issuecomment-202536977">this issue</a> on the Rust issue |
| tracker long ago. The reason is that even if we get called with arguments of |
| two distinct lifetimes, those two lifetimes have some intersection (the call |
| itself), and that intersection can be our value of <code>'a</code> that we use as the |
| common lifetime of our arguments. -nmatsakis)</p> |
| <h2 id="final-example"><a class="header" href="#final-example">Final example</a></h2> |
| <p>Let's look at one last example. We'll extend the previous one to have |
| a return type:</p> |
| <pre><code class="language-text">for<'a> fn(&'a u32, &'a u32) -> &'a u32 |
| <: |
| for<'b, 'c> fn(&'b u32, &'c u32) -> &'b u32 |
| </code></pre> |
| <p>Despite seeming very similar to the previous example, this case is going to get |
| an error. That's good: the problem is that we've gone from a fn that promises |
| to return one of its two arguments, to a fn that is promising to return the |
| first one. That is unsound. Let's see how it plays out.</p> |
| <p>First, we replace the bound region in the supertype with a placeholder:</p> |
| <pre><code class="language-text">for<'a> fn(&'a u32, &'a u32) -> &'a u32 |
| <: |
| fn(&'!1 u32, &'!2 u32) -> &'!1 u32 |
| </code></pre> |
| <p>Then we instantiate the subtype with existentials (in U2):</p> |
| <pre><code class="language-text">fn(&'?3 u32, &'?3 u32) -> &'?3 u32 |
| <: |
| fn(&'!1 u32, &'!2 u32) -> &'!1 u32 |
| </code></pre> |
| <p>And now we create the subtyping relationships:</p> |
| <pre><code class="language-text">&'!1 u32 <: &'?3 u32 // arg 1 |
| &'!2 u32 <: &'?3 u32 // arg 2 |
| &'?3 u32 <: &'!1 u32 // return type |
| </code></pre> |
| <p>And finally the outlives relationships. Here, let V1, V2, and V3 be the |
| variables we assign to <code>!1</code>, <code>!2</code>, and <code>?3</code> respectively:</p> |
| <pre><code class="language-text">V1: V3 |
| V2: V3 |
| V3: V1 |
| </code></pre> |
| <p>Those variables will have these initial values:</p> |
| <pre><code class="language-text">V1 in U1 = {placeholder(1)} |
| V2 in U2 = {placeholder(2)} |
| V3 in U2 = {} |
| </code></pre> |
| <p>Now because of the <code>V3: V1</code> constraint, we have to add <code>placeholder(1)</code> into <code>V3</code> (and |
| indeed it is visible from <code>V3</code>), so we get:</p> |
| <pre><code class="language-text">V3 in U2 = {placeholder(1)} |
| </code></pre> |
| <p>then we have this constraint <code>V2: V3</code>, so we wind up having to enlarge |
| <code>V2</code> to include <code>placeholder(1)</code> (which it can also see):</p> |
| <pre><code class="language-text">V2 in U2 = {placeholder(1), placeholder(2)} |
| </code></pre> |
| <p>Now constraint propagation is done, but when we check the outlives |
| relationships, we find that <code>V2</code> includes this new element <code>placeholder(1)</code>, |
| so we report an error.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="propagating-closure-constraints"><a class="header" href="#propagating-closure-constraints">Propagating closure constraints</a></h1> |
| <p>When we are checking the type tests and universal regions, we may come |
| across a constraint that we can't prove yet if we are in a closure |
| body! However, the necessary constraints may actually hold (we just |
| don't know it yet). Thus, if we are inside a closure, we just collect |
| all the constraints we can't prove yet and return them. Later, when we |
| are borrow check the MIR node that created the closure, we can also |
| check that these constraints hold. At that time, if we can't prove |
| they hold, we report an error.</p> |
| <h2 id="how-this-is-implemented"><a class="header" href="#how-this-is-implemented">How this is implemented</a></h2> |
| <p>While borrow-checking a closure inside of <code>RegionInferenceContext::solve</code> we separately try to propagate type-outlives and region-outlives constraints to the parent if we're unable to prove them locally.</p> |
| <h3 id="region-outlive-constraints"><a class="header" href="#region-outlive-constraints">Region-outlive constraints</a></h3> |
| <p>If <code>RegionInferenceContext::check_universal_regions</code> fails to prove some outlives constraint <code>'longer_fr: 'shorter_fr</code>, we try to propagate it in <code>fn try_propagate_universal_region_error</code>. Both these universal regions are either local to the closure or an external region.</p> |
| <p>In case <code>'longer_fr</code> is a local universal region, we search for the largest external region <code>'fr_minus</code> which is outlived by <code>'longer_fr</code>, i.e. <code>'longer_fr: 'fr_minus</code>. In case there are multiple such regions, we pick the <code>mutual_immediate_postdominator</code>: the fixpoint of repeatedly computing the GLB of all GLBs, see <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/transitive_relation/struct.TransitiveRelation.html#method.postdom_upper_bound">TransitiveRelation::postdom_upper_bound</a> for more details.</p> |
| <p>If <code>'fr_minus</code> exists we require it to outlive all non-local upper bounds of <code>'shorter_fr</code>. There will always be at least one non-local upper bound <code>'static</code>.</p> |
| <h3 id="type-outlive-constraints"><a class="header" href="#type-outlive-constraints">Type-outlive constraints</a></h3> |
| <p>Type-outlives constraints are proven in <code>check_type_tests</code>. This happens after computing the outlives graph, which is now immutable.</p> |
| <p>For all type tests we fail to prove via <code>fn eval_verify_bound</code> inside of the closure we call <code>try_promote_type_test</code>. A <code>TypeTest</code> represents a type-outlives bound <code>generic_kind: lower_bound</code> together with a <code>verify_bound</code>. If the <code>VerifyBound</code> holds for the <code>lower_bound</code>, the constraint is satisfied. <code>try_promote_type_test</code> does not care about the <code> verify_bound</code>.</p> |
| <p>It starts by calling <code>fn try_promote_type_test_subject</code>. This function takes the <code>GenericKind</code> and tries to transform it to a <code>ClosureOutlivesSubject</code> which is no longer references anything local to the closure. This is done by replacing all free regions in that type with either <code>'static</code> or region parameters which are equal to that free region. This operation fails if the <code>generic_kind</code> contains a region which cannot be replaced.</p> |
| <p>We then promote the <code>lower_bound</code> into the context of the caller. If the lower bound is equal to a placeholder, we replace it with <code>'static</code></p> |
| <p>We then look at all universal regions <code>uv</code> which are required to be outlived by <code>lower_bound</code>, i.e. for which borrow checking added region constraints. For each of these we then emit a <code>ClosureOutlivesRequirement</code> for all non-local universal regions which are known to outlive <code>uv</code>.</p> |
| <p>As we've already built the region graph of the closure at this point and separately check that it is consistent, we are also able to assume the outlive constraints <code>uv: lower_bound</code> here.</p> |
| <p>So if we have a type-outlives bounds we can't prove, e.g. <code>T: 'local_infer</code>, we use the region graph to go to universal variables <code>'a</code> with <code>'a: local_infer</code>. In case <code>'a</code> are local, we then use the assumed outlived constraints to go to non-local ones.</p> |
| <p>We then store the list of promoted type tests in the <code>BorrowCheckResults</code>. |
| We then apply them in while borrow-checking its parent in <code>TypeChecker::prove_closure_bounds</code>.</p> |
| <p>TODO: explain how exactly that works :3</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="reporting-region-errors"><a class="header" href="#reporting-region-errors">Reporting region errors</a></h1> |
| <p>TODO: we should discuss how to generate errors from the results of these analyses.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="two-phase-borrows"><a class="header" href="#two-phase-borrows">Two-phase borrows</a></h1> |
| <p>Two-phase borrows are a more permissive version of mutable borrows that allow |
| nested method calls such as <code>vec.push(vec.len())</code>. Such borrows first act as |
| shared borrows in a "reservation" phase and can later be "activated" into a |
| full mutable borrow.</p> |
| <p>Only certain implicit mutable borrows can be two-phase, any <code>&mut</code> or <code>ref mut</code> |
| in the source code is never a two-phase borrow. The cases where we generate a |
| two-phase borrow are:</p> |
| <ol> |
| <li>The autoref borrow when calling a method with a mutable reference receiver.</li> |
| <li>A mutable reborrow in function arguments.</li> |
| <li>The implicit mutable borrow in an overloaded compound assignment operator.</li> |
| </ol> |
| <p>To give some examples:</p> |
| <pre><pre class="playground"><code class="language-rust edition2018"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>// In the source code |
| |
| // Case 1: |
| let mut v = Vec::new(); |
| v.push(v.len()); |
| let r = &mut Vec::new(); |
| r.push(r.len()); |
| |
| // Case 2: |
| std::mem::replace(r, vec![1, r.len()]); |
| |
| // Case 3: |
| let mut x = std::num::Wrapping(2); |
| x += x; |
| <span class="boring">}</span></code></pre></pre> |
| <p>Expanding these enough to show the two-phase borrows:</p> |
| <pre><code class="language-rust ignore">// Case 1: |
| let mut v = Vec::new(); |
| let temp1 = &two_phase v; |
| let temp2 = v.len(); |
| Vec::push(temp1, temp2); |
| let r = &mut Vec::new(); |
| let temp3 = &two_phase *r; |
| let temp4 = r.len(); |
| Vec::push(temp3, temp4); |
| |
| // Case 2: |
| let temp5 = &two_phase *r; |
| let temp6 = vec![1, r.len()]; |
| std::mem::replace(temp5, temp6); |
| |
| // Case 3: |
| let mut x = std::num::Wrapping(2); |
| let temp7 = &two_phase x; |
| let temp8 = x; |
| std::ops::AddAssign::add_assign(temp7, temp8);</code></pre> |
| <p>Whether a borrow can be two-phase is tracked by a flag on the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/adjustment/enum.AutoBorrow.html"><code>AutoBorrow</code></a> |
| after type checking, which is then <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/cx/expr/trait.ToBorrowKind.html#method.to_borrow_kind">converted</a> to a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.BorrowKind.html"><code>BorrowKind</code></a> during MIR |
| construction.</p> |
| <p>Each two-phase borrow is assigned to a temporary that is only used once. As |
| such we can define:</p> |
| <ul> |
| <li>The point where the temporary is assigned to is called the <em>reservation</em> |
| point of the two-phase borrow.</li> |
| <li>The point where the temporary is used, which is effectively always a |
| function call, is called the <em>activation</em> point.</li> |
| </ul> |
| <p>The activation points are found using the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/borrow_set/struct.GatherBorrows.html"><code>GatherBorrows</code></a> visitor. The |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/borrow_set/struct.BorrowData.html"><code>BorrowData</code></a> then holds both the reservation and activation points for the |
| borrow.</p> |
| <h2 id="checking-two-phase-borrows"><a class="header" href="#checking-two-phase-borrows">Checking two-phase borrows</a></h2> |
| <p>Two-phase borrows are treated as if they were mutable borrows with the |
| following exceptions:</p> |
| <ol> |
| <li>At every location in the MIR we <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/struct.MirBorrowckCtxt.html#method.check_activations">check</a> if any two-phase borrows are |
| activated at this location. If a live two phase borrow is activated at a |
| location, then we check that there are no borrows that conflict with the |
| two-phase borrow.</li> |
| <li>At the reservation point we error if there are conflicting live <em>mutable</em> |
| borrows. And lint if there are any conflicting shared borrows.</li> |
| <li>Between the reservation and the activation point, the two-phase borrow acts |
| as a shared borrow. We determine (in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/path_utils/fn.is_active.html"><code>is_active</code></a>) if we're at such a point |
| by using the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/graph/dominators/struct.Dominators.html"><code>Dominators</code></a> for the MIR graph.</li> |
| <li>After the activation point, the two-phase borrow acts as a mutable borrow.</li> |
| </ol> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="closure-capture-inference"><a class="header" href="#closure-capture-inference">Closure Capture Inference</a></h1> |
| <p>This section describes how rustc handles closures. Closures in Rust are |
| effectively "desugared" into structs that contain the values they use (or |
| references to the values they use) from their creator's stack frame. rustc has |
| the job of figuring out which values a closure uses and how, so it can decide |
| whether to capture a given variable by shared reference, mutable reference, or |
| by move. rustc also has to figure out which of the closure traits (<a href="https://doc.rust-lang.org/std/ops/trait.Fn.html"><code>Fn</code></a>, |
| <a href="https://doc.rust-lang.org/std/ops/trait.FnMut.html"><code>FnMut</code></a>, or <a href="https://doc.rust-lang.org/std/ops/trait.FnOnce.html"><code>FnOnce</code></a>) a closure is capable of |
| implementing.</p> |
| <p>Let's start with a few examples:</p> |
| <h3 id="example-1"><a class="header" href="#example-1">Example 1</a></h3> |
| <p>To start, let's take a look at how the closure in the following example is desugared:</p> |
| <pre><pre class="playground"><code class="language-rust">fn closure(f: impl Fn()) { |
| f(); |
| } |
| |
| fn main() { |
| let x: i32 = 10; |
| closure(|| println!("Hi {}", x)); // The closure just reads x. |
| println!("Value of x after return {}", x); |
| }</code></pre></pre> |
| <p>Let's say the above is the content of a file called <code>immut.rs</code>. If we compile |
| <code>immut.rs</code> using the following command. The <a href="./mir/passes.html"><code>-Z dump-mir=all</code></a> flag will cause |
| <code>rustc</code> to generate and dump the <a href="./mir/index.html">MIR</a> to a directory called <code>mir_dump</code>.</p> |
| <pre><code class="language-console">> rustc +stage1 immut.rs -Z dump-mir=all |
| </code></pre> |
| <p>After we run this command, we will see a newly generated directory in our |
| current working directory called <code>mir_dump</code>, which will contain several files. |
| If we look at file <code>rustc.main.-------.mir_map.0.mir</code>, we will find, among |
| other things, it also contains this line:</p> |
| <pre><code class="language-rust ignore">_4 = &_1; |
| _3 = [closure@immut.rs:7:13: 7:36] { x: move _4 };</code></pre> |
| <p>Note that in the MIR examples in this chapter, <code>_1</code> is <code>x</code>.</p> |
| <p>Here in first line <code>_4 = &_1;</code>, the <code>mir_dump</code> tells us that <code>x</code> was borrowed |
| as an immutable reference. This is what we would hope as our closure just |
| reads <code>x</code>.</p> |
| <h3 id="example-2"><a class="header" href="#example-2">Example 2</a></h3> |
| <p>Here is another example:</p> |
| <pre><pre class="playground"><code class="language-rust">fn closure(mut f: impl FnMut()) { |
| f(); |
| } |
| |
| fn main() { |
| let mut x: i32 = 10; |
| closure(|| { |
| x += 10; // The closure mutates the value of x |
| println!("Hi {}", x) |
| }); |
| println!("Value of x after return {}", x); |
| }</code></pre></pre> |
| <pre><code class="language-rust ignore">_4 = &mut _1; |
| _3 = [closure@mut.rs:7:13: 10:6] { x: move _4 };</code></pre> |
| <p>This time along, in the line <code>_4 = &mut _1;</code>, we see that the borrow is changed to mutable borrow. |
| Fair enough! The closure increments <code>x</code> by 10.</p> |
| <h3 id="example-3"><a class="header" href="#example-3">Example 3</a></h3> |
| <p>One more example:</p> |
| <pre><pre class="playground"><code class="language-rust">fn closure(f: impl FnOnce()) { |
| f(); |
| } |
| |
| fn main() { |
| let x = vec![21]; |
| closure(|| { |
| drop(x); // Makes x unusable after the fact. |
| }); |
| // println!("Value of x after return {:?}", x); |
| }</code></pre></pre> |
| <pre><code class="language-rust ignore">_6 = [closure@move.rs:7:13: 9:6] { x: move _1 }; // bb16[3]: scope 1 at move.rs:7:13: 9:6</code></pre> |
| <p>Here, <code>x</code> is directly moved into the closure and the access to it will not be permitted after the |
| closure.</p> |
| <h2 id="inferences-in-the-compiler"><a class="header" href="#inferences-in-the-compiler">Inferences in the compiler</a></h2> |
| <p>Now let's dive into rustc code and see how all these inferences are done by the compiler.</p> |
| <p>Let's start with defining a term that we will be using quite a bit in the rest of the discussion - |
| <em>upvar</em>. An <strong>upvar</strong> is a variable that is local to the function where the closure is defined. So, |
| in the above examples, <strong>x</strong> will be an upvar to the closure. They are also sometimes referred to as |
| the <em>free variables</em> meaning they are not bound to the context of the closure. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_passes/upvars/index.html"><code>compiler/rustc_passes/src/upvars.rs</code></a> defines a query called <em>upvars_mentioned</em> |
| for this purpose.</p> |
| <p>Other than lazy invocation, one other thing that distinguishes a closure from a |
| normal function is that it can use the upvars. It borrows these upvars from its surrounding |
| context; therefore the compiler has to determine the upvar's borrow type. The compiler starts with |
| assigning an immutable borrow type and lowers the restriction (that is, changes it from |
| <strong>immutable</strong> to <strong>mutable</strong> to <strong>move</strong>) as needed, based on the usage. In the Example 1 above, the |
| closure only uses the variable for printing but does not modify it in any way and therefore, in the |
| <code>mir_dump</code>, we find the borrow type for the upvar <code>x</code> to be immutable. In example 2, however, the |
| closure modifies <code>x</code> and increments it by some value. Because of this mutation, the compiler, which |
| started off assigning <code>x</code> as an immutable reference type, has to adjust it as a mutable reference. |
| Likewise in the third example, the closure drops the vector and therefore this requires the variable |
| <code>x</code> to be moved into the closure. Depending on the borrow kind, the closure has to implement the |
| appropriate trait: <code>Fn</code> trait for immutable borrow, <code>FnMut</code> for mutable borrow, |
| and <code>FnOnce</code> for move semantics.</p> |
| <p>Most of the code related to the closure is in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/upvar/index.html"><code>compiler/rustc_hir_typeck/src/upvar.rs</code></a> file and the data structures are |
| declared in the file <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/index.html"><code>compiler/rustc_middle/src/ty/mod.rs</code></a>.</p> |
| <p>Before we go any further, let's discuss how we can examine the flow of control through the rustc |
| codebase. For closures specifically, set the <code>RUSTC_LOG</code> env variable as below and collect the |
| output in a file:</p> |
| <pre><code class="language-console">> RUSTC_LOG=rustc_hir_typeck::upvar rustc +stage1 -Z dump-mir=all \ |
| <.rs file to compile> 2> <file where the output will be dumped> |
| </code></pre> |
| <p>This uses the stage1 compiler and enables <code>debug!</code> logging for the |
| <code>rustc_hir_typeck::upvar</code> module.</p> |
| <p>The other option is to step through the code using lldb or gdb.</p> |
| <ol> |
| <li><code>rust-lldb build/host/stage1/bin/rustc test.rs</code></li> |
| <li>In lldb: |
| <ol> |
| <li><code>b upvar.rs:134</code> // Setting the breakpoint on a certain line in the upvar.rs file</li> |
| <li><code>r</code> // Run the program until it hits the breakpoint</li> |
| </ol> |
| </li> |
| </ol> |
| <p>Let's start with <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/upvar/index.html"><code>upvar.rs</code></a>. This file has something called |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/expr_use_visitor/struct.ExprUseVisitor.html"><code>euv::ExprUseVisitor</code></a> which walks the source of the closure and |
| invokes a callback for each upvar that is borrowed, mutated, or moved.</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let mut x = vec![21]; |
| let _cl = || { |
| let y = x[0]; // 1. |
| x[0] += 1; // 2. |
| }; |
| }</code></pre></pre> |
| <p>In the above example, our visitor will be called twice, for the lines marked 1 and 2, once for a |
| shared borrow and another one for a mutable borrow. It will also tell us what was borrowed.</p> |
| <p>The callbacks are defined by implementing the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/expr_use_visitor/trait.Delegate.html"><code>Delegate</code></a> trait. The |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/upvar/struct.InferBorrowKind.html"><code>InferBorrowKind</code></a> type implements <code>Delegate</code> and keeps a map that |
| records for each upvar which mode of capture was required. The modes of capture |
| can be <code>ByValue</code> (moved) or <code>ByRef</code> (borrowed). For <code>ByRef</code> borrows, the possible |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.BorrowKind.html"><code>BorrowKind</code></a>s are <code>ImmBorrow</code>, <code>UniqueImmBorrow</code>, <code>MutBorrow</code> as defined in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/index.html"><code>compiler/rustc_middle/src/ty/mod.rs</code></a>.</p> |
| <p><code>Delegate</code> defines a few different methods (the different callbacks): |
| <strong>consume</strong> for <em>move</em> of a variable, <strong>borrow</strong> for a <em>borrow</em> of some kind |
| (shared or mutable), and <strong>mutate</strong> when we see an <em>assignment</em> of something.</p> |
| <p>All of these callbacks have a common argument <em>cmt</em> which stands for Category, |
| Mutability and Type and is defined in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/expr_use_visitor/index.html"><code>compiler/rustc_hir_typeck/src/expr_use_visitor.rs</code></a>. Borrowing from the code |
| comments, "<code>cmt</code> is a complete categorization of a value indicating where it |
| originated and how it is located, as well as the mutability of the memory in |
| which the value is stored". Based on the callback (consume, borrow etc.), we |
| will call the relevant <code>adjust_upvar_borrow_kind_for_<something></code> and pass the |
| <code>cmt</code> along. Once the borrow type is adjusted, we store it in the table, which |
| basically says what borrows were made for each closure.</p> |
| <pre><code class="language-rust ignore">self.tables |
| .borrow_mut() |
| .upvar_capture_map |
| .extend(delegate.adjust_upvar_captures);</code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="async-closurescoroutine-closures"><a class="header" href="#async-closurescoroutine-closures">Async closures/"coroutine-closures"</a></h1> |
| <p>Please read <a href="https://rust-lang.github.io/rfcs/3668-async-closures.html">RFC 3668</a> to understand the general motivation of the feature. This is a very technical and somewhat "vertical" chapter; ideally we'd split this and sprinkle it across all the relevant chapters, but for the purposes of understanding async closures <em>holistically</em>, I've put this together all here in one chapter.</p> |
| <h2 id="coroutine-closures----a-technical-deep-dive"><a class="header" href="#coroutine-closures----a-technical-deep-dive">Coroutine-closures -- a technical deep dive</a></h2> |
| <p>Coroutine-closures are a generalization of async closures, being special syntax for closure expressions which return a coroutine, notably one that is allowed to capture from the closure's upvars.</p> |
| <p>For now, the only usable kind of coroutine-closure is the async closure, and supporting async closures is the extent of this PR. We may eventually support <code>gen || {}</code>, etc., and most of the problems and curiosities described in this document apply to all coroutine-closures in general.</p> |
| <p>As a consequence of the code being somewhat general, this document may flip between calling them "async closures" and "coroutine-closures". The future that is returned by the async closure will generally be called the "coroutine" or the "child coroutine".</p> |
| <h3 id="hir-1"><a class="header" href="#hir-1">HIR</a></h3> |
| <p>Async closures (and in the future, other coroutine flavors such as <code>gen</code>) are represented in HIR as a <code>hir::Closure</code>. |
| The closure-kind of the <code>hir::Closure</code> is <code>ClosureKind::CoroutineClosure(_)</code><sup class="footnote-reference" id="fr-k1-1"><a href="#footnote-k1">1</a></sup>, which wraps an async block, which is also represented in HIR as a <code>hir::Closure</code>. |
| The closure-kind of the async block is <code>ClosureKind::Closure(CoroutineKind::Desugared(_, CoroutineSource::Closure))</code><sup class="footnote-reference" id="fr-k2-1"><a href="#footnote-k2">2</a></sup>.</p> |
| <p>Like <code>async fn</code>, when lowering an async closure's body, we need to unconditionally move all of the closures arguments into the body so they are captured. This is handled by <code>lower_coroutine_body_with_moved_arguments</code><sup class="footnote-reference" id="fr-l1-1"><a href="#footnote-l1">3</a></sup>. The only notable quirk with this function is that the async block we end up generating as a capture kind of <code>CaptureBy::ByRef</code><sup class="footnote-reference" id="fr-l2-1"><a href="#footnote-l2">4</a></sup>. We later force all of the <em>closure args</em> to be captured by-value<sup class="footnote-reference" id="fr-l3-1"><a href="#footnote-l3">5</a></sup>, but we don't want the <em>whole</em> async block to act as if it were an <code>async move</code>, since that would defeat the purpose of the self-borrowing of an async closure.</p> |
| <h3 id="rustc_middlety-representation"><a class="header" href="#rustc_middlety-representation"><code>rustc_middle::ty</code> Representation</a></h3> |
| <p>For the purposes of keeping the implementation mostly future-compatible (i.e. with gen <code>|| {}</code> and <code>async gen || {}</code>), most of this section calls async closures "coroutine-closures".</p> |
| <p>The main thing that this PR introduces is a new <code>TyKind</code> called <code>CoroutineClosure</code><sup class="footnote-reference" id="fr-t1-1"><a href="#footnote-t1">6</a></sup> and corresponding variants on other relevant enums in typeck and borrowck (<code>UpvarArgs</code>, <code>DefiningTy</code>, <code>AggregateKind</code>).</p> |
| <p>We introduce a new <code>TyKind</code> instead of generalizing the existing <code>TyKind::Closure</code> due to major representational differences in the type. The major differences between <code>CoroutineClosure</code>s can be explored by first inspecting the <code>CoroutineClosureArgsParts</code>, which is the "unpacked" representation of the coroutine-closure's generics.</p> |
| <h4 id="similarities-to-closures"><a class="header" href="#similarities-to-closures">Similarities to closures</a></h4> |
| <p>Like a closure, we have <code>parent_args</code>, a <code>closure_kind_ty</code>, and a <code>tupled_upvars_ty</code>. These represent the same thing as their closure counterparts; namely: the generics inherited from the body that the closure is defined in, the maximum "calling capability" of the closure (i.e. must it be consumed to be called, like <code>FnOnce</code>, or can it be called by-ref), and the captured upvars of the closure itself.</p> |
| <h4 id="the-signature"><a class="header" href="#the-signature">The signature</a></h4> |
| <p>A traditional closure has a <code>fn_sig_as_fn_ptr_ty</code> which it uses to represent the signature of the closure. In contrast, we store the signature of a coroutine closure in a somewhat "exploded" way, since coroutine-closures have <em>two</em> signatures depending on what <code>AsyncFn*</code> trait you call it with (see below sections).</p> |
| <p>Conceptually, the coroutine-closure may be thought as containing several different signature types depending on whether it is being called by-ref or by-move.</p> |
| <p>To conveniently recreate both of these signatures, the <code>signature_parts_ty</code> stores all of the relevant parts of the coroutine returned by this coroutine-closure. This signature parts type will have the general shape of <code>fn(tupled_inputs, resume_ty) -> (return_ty, yield_ty)</code>, where <code>resume_ty</code>, <code>return_ty</code>, and <code>yield_ty</code> are the respective types for the <em>coroutine</em> returned by the coroutine-closure<sup class="footnote-reference" id="fr-c1-1"><a href="#footnote-c1">7</a></sup>.</p> |
| <p>The compiler mainly deals with the <code>CoroutineClosureSignature</code> type<sup class="footnote-reference" id="fr-c2-1"><a href="#footnote-c2">8</a></sup>, which is created by extracting the relevant types out of the <code>fn()</code> ptr type described above, and which exposes methods that can be used to construct the <em>coroutine</em> that the coroutine-closure ultimately returns.</p> |
| <h4 id="the-data-we-need-to-carry-along-to-construct-a-coroutine-return-type"><a class="header" href="#the-data-we-need-to-carry-along-to-construct-a-coroutine-return-type">The data we need to carry along to construct a <code>Coroutine</code> return type</a></h4> |
| <p>Along with the data stored in the signature, to construct a <code>TyKind::Coroutine</code> to return, we also need to store the "witness" of the coroutine.</p> |
| <p>So what about the upvars of the <code>Coroutine</code> that is returned? Well, for <code>AsyncFnOnce</code> (i.e. call-by-move), this is simply the same upvars that the coroutine returns. But for <code>AsyncFnMut</code>/<code>AsyncFn</code>, the coroutine that is returned from the coroutine-closure borrows data from the coroutine-closure with a given "environment" lifetime<sup class="footnote-reference" id="fr-c3-1"><a href="#footnote-c3">9</a></sup>. This corresponds to the <code>&self</code> lifetime<sup class="footnote-reference" id="fr-c4-1"><a href="#footnote-c4">10</a></sup> on the <code>AsyncFnMut</code>/<code>AsyncFn</code> call signature, and the GAT lifetime of the <code>ByRef</code><sup class="footnote-reference" id="fr-c5-1"><a href="#footnote-c5">11</a></sup>.</p> |
| <h4 id="actually-getting-the-coroutine-return-types"><a class="header" href="#actually-getting-the-coroutine-return-types">Actually getting the coroutine return type(s)</a></h4> |
| <p>To most easily construct the <code>Coroutine</code> that a coroutine-closure returns, you can use the <code>to_coroutine_given_kind_and_upvars</code><sup class="footnote-reference" id="fr-helper-1"><a href="#footnote-helper">12</a></sup> helper on <code>CoroutineClosureSignature</code>, which can be acquired from the <code>CoroutineClosureArgs</code>.</p> |
| <p>Most of the args to that function will be components that you can get out of the <code>CoroutineArgs</code>, except for the <code>goal_kind: ClosureKind</code> which controls which flavor of coroutine to return based off of the <code>ClosureKind</code> passed in -- i.e. it will prepare the by-ref coroutine if <code>ClosureKind::Fn | ClosureKind::FnMut</code>, and the by-move coroutine if <code>ClosureKind::FnOnce</code>.</p> |
| <h3 id="trait-hierarchy"><a class="header" href="#trait-hierarchy">Trait Hierarchy</a></h3> |
| <p>We introduce a parallel hierarchy of <code>Fn*</code> traits that are implemented for . The motivation for the introduction was covered in a blog post: <a href="https://hackmd.io/@compiler-errors/async-closures">Async Closures</a>.</p> |
| <p>All currently-stable callable types (i.e., closures, function items, function pointers, and <code>dyn Fn*</code> trait objects) automatically implement <code>AsyncFn*() -> T</code> if they implement <code>Fn*() -> Fut</code> for some output type <code>Fut</code>, and <code>Fut</code> implements <code>Future<Output = T></code><sup class="footnote-reference" id="fr-tr1-1"><a href="#footnote-tr1">13</a></sup>.</p> |
| <p>Async closures implement <code>AsyncFn*</code> as their bodies permit; i.e. if they end up using upvars in a way that is compatible (i.e. if they consume or mutate their upvars, it may affect whether they implement <code>AsyncFn</code> and <code>AsyncFnMut</code>...)</p> |
| <h4 id="lending"><a class="header" href="#lending">Lending</a></h4> |
| <p>We may in the future move <code>AsyncFn*</code> onto a more general set of <code>LendingFn*</code> traits; however, there are some concrete technical implementation details that limit our ability to use <code>LendingFn</code> ergonomically in the compiler today. These have to do with:</p> |
| <ul> |
| <li>Closure signature inference.</li> |
| <li>Limitations around higher-ranked trait bounds.</li> |
| <li>Shortcomings with error messages.</li> |
| </ul> |
| <p>These limitations, plus the fact that the underlying trait should have no effect on the user experience of async closures and async <code>Fn</code> trait bounds, leads us to <code>AsyncFn*</code> for now. To ensure we can eventually move to these more general traits, the precise <code>AsyncFn*</code> trait definitions (including the associated types) are left as an implementation detail.</p> |
| <h4 id="when-do-async-closures-implement-the-regular-fn-traits"><a class="header" href="#when-do-async-closures-implement-the-regular-fn-traits">When do async closures implement the regular <code>Fn*</code> traits?</a></h4> |
| <p>We mention above that "regular" callable types can implement <code>AsyncFn*</code>, but the reverse question exists of "can async closures implement <code>Fn*</code> too"? The short answer is "when it's valid", i.e. when the coroutine that would have been returned from <code>AsyncFn</code>/<code>AsyncFnMut</code> does not actually have any upvars that are "lent" from the parent coroutine-closure.</p> |
| <p>See the "follow-up: when do..." section below for an elaborated answer. The full answer describes a pretty interesting and hopefully thorough heuristic that is used to ensure that most async closures "just work".</p> |
| <h3 id="tale-of-two-bodies"><a class="header" href="#tale-of-two-bodies">Tale of two bodies...</a></h3> |
| <p>When async closures are called with <code>AsyncFn</code>/<code>AsyncFnMut</code>, they return a coroutine that borrows from the closure. However, when they are called via <code>AsyncFnOnce</code>, we consume that closure, and cannot return a coroutine that borrows from data that is now dropped.</p> |
| <p>To work around this limitation, we synthesize a separate by-move MIR body for calling <code>AsyncFnOnce::call_once</code> on a coroutine-closure that can be called by-ref.</p> |
| <p>This body operates identically to the "normal" coroutine returned from calling the coroutine-closure, except for the fact that it has a different set of upvars, since we must <em>move</em> the captures from the parent coroutine-closure into the child coroutine.</p> |
| <h4 id="synthesizing-the-by-move-body"><a class="header" href="#synthesizing-the-by-move-body">Synthesizing the by-move body</a></h4> |
| <p>When we want to access the by-move body of the coroutine returned by a coroutine-closure, we can do so via the <code>coroutine_by_move_body_def_id</code><sup class="footnote-reference" id="fr-b1-1"><a href="#footnote-b1">14</a></sup> query.</p> |
| <p>This query synthesizes a new MIR body by copying the MIR body of the coroutine and inserting additional derefs and field projections<sup class="footnote-reference" id="fr-b2-1"><a href="#footnote-b2">15</a></sup> to preserve the semantics of the body.</p> |
| <p>Since we've synthesized a new def id, this query is also responsible for feeding a ton of other relevant queries for the MIR body. This query is <code>ensure()</code>d<sup class="footnote-reference" id="fr-b3-1"><a href="#footnote-b3">16</a></sup> during the <code>mir_promoted</code> query, since it operates on the <em>built</em> mir of the coroutine.</p> |
| <h3 id="closure-signature-inference"><a class="header" href="#closure-signature-inference">Closure signature inference</a></h3> |
| <p>The closure signature inference algorithm for async closures is a bit more complicated than the inference algorithm for "traditional" closures. Like closures, we iterate through all of the clauses that may be relevant (for the expectation type passed in)<sup class="footnote-reference" id="fr-deduce1-1"><a href="#footnote-deduce1">17</a></sup>.</p> |
| <p>To extract a signature, we consider two situations:</p> |
| <ul> |
| <li>Projection predicates with <code>AsyncFnOnce::Output</code>, which we will use to extract the inputs and output type for the closure. This corresponds to the situation that there was a <code>F: AsyncFn*() -> T</code> bound<sup class="footnote-reference" id="fr-deduce2-1"><a href="#footnote-deduce2">18</a></sup>.</li> |
| <li>Projection predicates with <code>FnOnce::Output</code>, which we will use to extract the inputs. For the output, we also try to deduce an output by looking for relevant <code>Future::Output</code> projection predicates. This corresponds to the situation that there was an <code>F: Fn*() -> T, T: Future<Output = U></code> bound.<sup class="footnote-reference" id="fr-deduce3-1"><a href="#footnote-deduce3">19</a></sup> |
| <ul> |
| <li>If there is no <code>Future</code> bound, we simply use a fresh infer var for the output. This corresponds to the case where one can pass an async closure to a combinator function like <code>Option::map</code>.<sup class="footnote-reference" id="fr-deduce4-1"><a href="#footnote-deduce4">20</a></sup></li> |
| </ul> |
| </li> |
| </ul> |
| <p>We support the latter case simply to make it easier for users to simply drop-in <code>async || {}</code> syntax, even when they're calling an API that was designed before first-class <code>AsyncFn*</code> traits were available.</p> |
| <h4 id="calling-a-closure-before-its-kind-has-been-inferred"><a class="header" href="#calling-a-closure-before-its-kind-has-been-inferred">Calling a closure before its kind has been inferred</a></h4> |
| <p>We defer<sup class="footnote-reference" id="fr-call1-1"><a href="#footnote-call1">21</a></sup> the computation of a coroutine-closure's "kind" (i.e. its maximum calling mode: <code>AsyncFnOnce</code>/<code>AsyncFnMut</code>/<code>AsyncFn</code>) until the end of typeck. However, since we want to be able to call that coroutine-closure before the end of typeck, we need to come up with the return type of the coroutine-closure before that.</p> |
| <p>Unlike regular closures, whose return type does not change depending on what <code>Fn*</code> trait we call it with, coroutine-closures <em>do</em> end up returning different coroutine types depending on the flavor of <code>AsyncFn*</code> trait used to call it.</p> |
| <p>Specifically, while the def-id of the returned coroutine does not change, the upvars<sup class="footnote-reference" id="fr-call2-1"><a href="#footnote-call2">22</a></sup> (which are either borrowed or moved from the parent coroutine-closure) and the coroutine-kind<sup class="footnote-reference" id="fr-call3-1"><a href="#footnote-call3">23</a></sup> are dependent on the calling mode.</p> |
| <p>We introduce a <code>AsyncFnKindHelper</code> trait which allows us to defer the question of "does this coroutine-closure support this calling mode"<sup class="footnote-reference" id="fr-helper1-1"><a href="#footnote-helper1">24</a></sup> via a trait goal, and "what are the tupled upvars of this calling mode"<sup class="footnote-reference" id="fr-helper2-1"><a href="#footnote-helper2">25</a></sup> via an associated type, which can be computed by appending the input types of the coroutine-closure to either the upvars or the "by ref" upvars computed during upvar analysis.</p> |
| <h4 id="ok-so-why"><a class="header" href="#ok-so-why">Ok, so why?</a></h4> |
| <p>This seems a bit roundabout and complex, and I admit that it is. But let's think of the "do nothing" alternative -- we could instead mark all <code>AsyncFn*</code> goals as ambiguous until upvar analysis, at which point we would know exactly what to put into the upvars of the coroutine we return. However, this is actually <em>very</em> detrimental to inference in the program, since it means that programs like this would not be valid:</p> |
| <pre><code class="language-rust ignore">let c = async || -> String { .. }; |
| let s = c().await; |
| // ^^^ If we can't project `<{c} as AsyncFn>::call()` to a coroutine, then the `IntoFuture::into_future` call inside of the `.await` stalls, and the type of `s` is left unconstrained as an infer var. |
| s.as_bytes(); |
| // ^^^ That means we can't call any methods on the awaited return of a coroutine-closure, like... at all!</code></pre> |
| <p>So <em>instead</em>, we use this alias (in this case, a projection: <code>AsyncFnKindHelper::Upvars<'env, ...></code>) to delay the computation of the <em>tupled upvars</em> and give us something to put in its place, while still allowing us to return a <code>TyKind::Coroutine</code> (which is a rigid type) and we may successfully confirm the built-in traits we need (in our case, <code>Future</code>), since the <code>Future</code> implementation doesn't depend on the upvars at all.</p> |
| <h3 id="upvar-analysis"><a class="header" href="#upvar-analysis">Upvar analysis</a></h3> |
| <p>By and large, the upvar analysis for coroutine-closures and their child coroutines proceeds like normal upvar analysis. However, there are several interesting bits that happen to account for async closures' special natures:</p> |
| <h4 id="forcing-all-inputs-to-be-captured"><a class="header" href="#forcing-all-inputs-to-be-captured">Forcing all inputs to be captured</a></h4> |
| <p>Like async fn, all input arguments are captured. We explicitly force<sup class="footnote-reference" id="fr-f1-1"><a href="#footnote-f1">26</a></sup> all of these inputs to be captured by move so that the future coroutine returned by async closures does not depend on whether the input is <em>used</em> by the body or not, which would impart an interesting semver hazard.</p> |
| <h4 id="computing-the-by-ref-captures"><a class="header" href="#computing-the-by-ref-captures">Computing the by-ref captures</a></h4> |
| <p>For a coroutine-closure that supports <code>AsyncFn</code>/<code>AsyncFnMut</code>, we must also compute the relationship between the captures of the coroutine-closure and its child coroutine. Specifically, the coroutine-closure may <code>move</code> a upvar into its captures, but the coroutine may only borrow that upvar.</p> |
| <p>We compute the "<code>coroutine_captures_by_ref_ty</code>" by looking at all of the child coroutine's captures and comparing them to the corresponding capture of the parent coroutine-closure<sup class="footnote-reference" id="fr-br1-1"><a href="#footnote-br1">27</a></sup>. This <code>coroutine_captures_by_ref_ty</code> ends up being represented as a <code>for<'env> fn() -> captures...</code> type, with the additional binder lifetime representing the "<code>&self</code>" lifetime of calling <code>AsyncFn::async_call</code> or <code>AsyncFnMut::async_call_mut</code>. We instantiate that binder later when actually calling the methods.</p> |
| <p>Note that not every by-ref capture from the parent coroutine-closure results in a "lending" borrow. See the <strong>Follow-up: When do async closures implement the regular <code>Fn*</code> traits?</strong> section below for more details, since this intimately influences whether or not the coroutine-closure is allowed to implement the <code>Fn*</code> family of traits.</p> |
| <h4 id="by-move-body--fnonce-quirk"><a class="header" href="#by-move-body--fnonce-quirk">By-move body + <code>FnOnce</code> quirk</a></h4> |
| <p>There are several situations where the closure upvar analysis ends up inferring upvars for the coroutine-closure's child coroutine that are too relaxed, and end up resulting in borrow-checker errors. This is best illustrated via examples. For example, given:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>fn force_fnonce<T: async FnOnce()>(t: T) -> T { t } |
| |
| let x = String::new(); |
| let c = force_fnonce(async move || { |
| println!("{x}"); |
| }); |
| <span class="boring">}</span></code></pre></pre> |
| <p><code>x</code> will be moved into the coroutine-closure, but the coroutine that is returned would only borrow <code>&x</code>. However, since <code>force_fnonce</code> forces the coroutine-closure to <code>AsyncFnOnce</code>, which is not <em>lending</em>, we must force the capture to happen by-move<sup class="footnote-reference" id="fr-bm1-1"><a href="#footnote-bm1">28</a></sup>.</p> |
| <p>Similarly:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>let x = String::new(); |
| let y = String::new(); |
| let c = async move || { |
| drop(y); |
| println!("{x}"); |
| }; |
| <span class="boring">}</span></code></pre></pre> |
| <p><code>x</code> will be moved into the coroutine-closure, but the coroutine that is returned would only borrow <code>&x</code>. However, since we also capture <code>y</code> and drop it, the coroutine-closure is forced to be <code>AsyncFnOnce</code>. We must also force the capture of <code>x</code> to happen by-move. To determine this situation in particular, since unlike the last example the coroutine-kind's closure-kind has not yet been constrained, we must analyze the body of the coroutine-closure to see if how all of the upvars are used, to determine if they've been used in a way that is "consuming" -- i.e. that would force it to <code>FnOnce</code><sup class="footnote-reference" id="fr-bm2-1"><a href="#footnote-bm2">29</a></sup>.</p> |
| <h4 id="follow-up-when-do-async-closures-implement-the-regular-fn-traits"><a class="header" href="#follow-up-when-do-async-closures-implement-the-regular-fn-traits">Follow-up: When do async closures implement the regular <code>Fn*</code> traits?</a></h4> |
| <p>Well, first of all, all async closures implement <code>FnOnce</code> since they can always be called <em>at least once</em>.</p> |
| <p>For <code>Fn</code>/<code>FnMut</code>, the detailed answer involves answering a related question: is the coroutine-closure lending? Because if it is, then it cannot implement the non-lending <code>Fn</code>/<code>FnMut</code> traits.</p> |
| <p>Determining when the coroutine-closure must <em>lend</em> its upvars is implemented in the <code>should_reborrow_from_env_of_parent_coroutine_closure</code> helper function<sup class="footnote-reference" id="fr-u1-1"><a href="#footnote-u1">30</a></sup>. Specifically, this needs to happen in two places:</p> |
| <ol> |
| <li>Are we borrowing data owned by the parent closure? We can determine if that is the case by checking if the parent capture is by-move, EXCEPT if we apply a deref projection, which means we're reborrowing a reference that we captured by-move.</li> |
| </ol> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>let x = &1i32; // Let's call this lifetime `'1`. |
| let c = async move || { |
| println!("{:?}", *x); |
| // Even though the inner coroutine borrows by ref, we're only capturing `*x`, |
| // not `x`, so the inner closure is allowed to reborrow the data for `'1`. |
| }; |
| <span class="boring">}</span></code></pre></pre> |
| <ol start="2"> |
| <li>If a coroutine is mutably borrowing from a parent capture, then that mutable borrow cannot live for longer than either the parent <em>or</em> the borrow that we have on the original upvar. Therefore we always need to borrow the child capture with the lifetime of the parent coroutine-closure's env.</li> |
| </ol> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>let mut x = 1i32; |
| let c = async || { |
| x = 1; |
| // The parent borrows `x` for some `&'1 mut i32`. |
| // However, when we call `c()`, we implicitly autoref for the signature of |
| // `AsyncFnMut::async_call_mut`. Let's call that lifetime `'call`. Since |
| // the maximum that `&'call mut &'1 mut i32` can be reborrowed is `&'call mut i32`, |
| // the inner coroutine should capture w/ the lifetime of the coroutine-closure. |
| }; |
| <span class="boring">}</span></code></pre></pre> |
| <p>If either of these cases apply, then we should capture the borrow with the lifetime of the parent coroutine-closure's env. Luckily, if this function is not correct, then the program is not unsound, since we still borrowck and validate the choices made from this function -- the only side-effect is that the user may receive unnecessary borrowck errors.</p> |
| <h3 id="instance-resolution"><a class="header" href="#instance-resolution">Instance resolution</a></h3> |
| <p>If a coroutine-closure has a closure-kind of <code>FnOnce</code>, then its <code>AsyncFnOnce::call_once</code> and <code>FnOnce::call_once</code> implementations resolve to the coroutine-closure's body<sup class="footnote-reference" id="fr-res1-1"><a href="#footnote-res1">31</a></sup>, and the <code>Future::poll</code> of the coroutine that gets returned resolves to the body of the child closure.</p> |
| <p>If a coroutine-closure has a closure-kind of <code>FnMut</code>/<code>Fn</code>, then the same applies to <code>AsyncFn</code> and the corresponding <code>Future</code> implementation of the coroutine that gets returned.<sup class="footnote-reference" id="fr-res1-2"><a href="#footnote-res1">31</a></sup> However, we use a MIR shim to generate the implementation of <code>AsyncFnOnce::call_once</code>/<code>FnOnce::call_once</code><sup class="footnote-reference" id="fr-res2-1"><a href="#footnote-res2">32</a></sup>, and <code>Fn::call</code>/<code>FnMut::call_mut</code> instances if they exist<sup class="footnote-reference" id="fr-res3-1"><a href="#footnote-res3">33</a></sup>.</p> |
| <p>This is represented by the <code>ConstructCoroutineInClosureShim</code><sup class="footnote-reference" id="fr-i1-1"><a href="#footnote-i1">34</a></sup>. The <code>receiver_by_ref</code> bool will be true if this is the instance of <code>Fn::call</code>/<code>FnMut::call_mut</code>.<sup class="footnote-reference" id="fr-i2-1"><a href="#footnote-i2">35</a></sup> The coroutine that all of these instances returns corresponds to the by-move body we will have synthesized by this point.<sup class="footnote-reference" id="fr-i3-1"><a href="#footnote-i3">36</a></sup></p> |
| <h3 id="borrow-checking"><a class="header" href="#borrow-checking">Borrow-checking</a></h3> |
| <p>It turns out that borrow-checking async closures is pretty straightforward. After adding a new <code>DefiningTy::CoroutineClosure</code><sup class="footnote-reference" id="fr-bck1-1"><a href="#footnote-bck1">37</a></sup> variant, and teaching borrowck how to generate the signature of the coroutine-closure<sup class="footnote-reference" id="fr-bck2-1"><a href="#footnote-bck2">38</a></sup>, borrowck proceeds totally fine.</p> |
| <p>One thing to note is that we don't borrow-check the synthetic body we make for by-move coroutines, since by construction (and the validity of the by-ref coroutine body it was derived from) it must be valid.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-k1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/expr.rs#L1147">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/expr.rs#L1147</a> <a href="#fr-k1-1">↩</a></p> |
| </li> |
| <li id="footnote-k2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/expr.rs#L1117">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/expr.rs#L1117</a> <a href="#fr-k2-1">↩</a></p> |
| </li> |
| <li id="footnote-l1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/item.rs#L1096-L1100">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/item.rs#L1096-L1100</a> <a href="#fr-l1-1">↩</a></p> |
| </li> |
| <li id="footnote-l2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/item.rs#L1276-L1279">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_ast_lowering/src/item.rs#L1276-L1279</a> <a href="#fr-l2-1">↩</a></p> |
| </li> |
| <li id="footnote-l3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/upvar.rs#L250-L256">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/upvar.rs#L250-L256</a> <a href="#fr-l3-1">↩</a></p> |
| </li> |
| <li id="footnote-t1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind.rs#L163-L168">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind.rs#L163-L168</a> <a href="#fr-t1-1">↩</a></p> |
| </li> |
| <li id="footnote-c1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L221-L229">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L221-L229</a> <a href="#fr-c1-1">↩</a></p> |
| </li> |
| <li id="footnote-c2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L362">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L362</a> <a href="#fr-c2-1">↩</a></p> |
| </li> |
| <li id="footnote-c3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L447-L455">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L447-L455</a> <a href="#fr-c3-1">↩</a></p> |
| </li> |
| <li id="footnote-c4"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/library/core/src/ops/async_function.rs#L36">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/library/core/src/ops/async_function.rs#L36</a> <a href="#fr-c4-1">↩</a></p> |
| </li> |
| <li id="footnote-c5"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/library/core/src/ops/async_function.rs#L30">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/library/core/src/ops/async_function.rs#L30</a> <a href="#fr-c5-1">↩</a></p> |
| </li> |
| <li id="footnote-helper"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L419">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_type_ir/src/ty_kind/closure.rs#L419</a> <a href="#fr-helper-1">↩</a></p> |
| </li> |
| <li id="footnote-tr1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs#L404-L409">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs#L404-L409</a> <a href="#fr-tr1-1">↩</a></p> |
| </li> |
| <li id="footnote-b1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs#L1-L70">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs#L1-L70</a> <a href="#fr-b1-1">↩</a></p> |
| </li> |
| <li id="footnote-b2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs#L131-L195">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs#L131-L195</a> <a href="#fr-b2-1">↩</a></p> |
| </li> |
| <li id="footnote-b3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_mir_transform/src/lib.rs#L339-L342">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_mir_transform/src/lib.rs#L339-L342</a> <a href="#fr-b3-1">↩</a></p> |
| </li> |
| <li id="footnote-deduce1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L345-L362">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L345-L362</a> <a href="#fr-deduce1-1">↩</a></p> |
| </li> |
| <li id="footnote-deduce2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L486-L487">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L486-L487</a> <a href="#fr-deduce2-1">↩</a></p> |
| </li> |
| <li id="footnote-deduce3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L517-L534">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L517-L534</a> <a href="#fr-deduce3-1">↩</a></p> |
| </li> |
| <li id="footnote-deduce4"> |
| <p><a href="https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L575-L590">https://github.com/rust-lang/rust/blob/5ca0e9fa9b2f92b463a0a2b0b34315e09c0b7236/compiler/rustc_hir_typeck/src/closure.rs#L575-L590</a> <a href="#fr-deduce4-1">↩</a></p> |
| </li> |
| <li id="footnote-call1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_hir_typeck/src/callee.rs#L169-L210">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_hir_typeck/src/callee.rs#L169-L210</a> <a href="#fr-call1-1">↩</a></p> |
| </li> |
| <li id="footnote-call2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_type_ir/src/ty_kind/closure.rs#L574-L576">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_type_ir/src/ty_kind/closure.rs#L574-L576</a> <a href="#fr-call2-1">↩</a></p> |
| </li> |
| <li id="footnote-call3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_type_ir/src/ty_kind/closure.rs#L554-L563">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_type_ir/src/ty_kind/closure.rs#L554-L563</a> <a href="#fr-call3-1">↩</a></p> |
| </li> |
| <li id="footnote-helper1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/library/core/src/ops/async_function.rs#L135-L144">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/library/core/src/ops/async_function.rs#L135-L144</a> <a href="#fr-helper1-1">↩</a></p> |
| </li> |
| <li id="footnote-helper2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/library/core/src/ops/async_function.rs#L146-L154">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/library/core/src/ops/async_function.rs#L146-L154</a> <a href="#fr-helper2-1">↩</a></p> |
| </li> |
| <li id="footnote-f1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L250-L259">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L250-L259</a> <a href="#fr-f1-1">↩</a></p> |
| </li> |
| <li id="footnote-br1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L375-L471">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L375-L471</a> <a href="#fr-br1-1">↩</a></p> |
| </li> |
| <li id="footnote-bm1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L211-L248">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L211-L248</a> <a href="#fr-bm1-1">↩</a></p> |
| </li> |
| <li id="footnote-bm2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L532-L539">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L532-L539</a> <a href="#fr-bm2-1">↩</a></p> |
| </li> |
| <li id="footnote-u1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L1818-L1860">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_hir_typeck/src/upvar.rs#L1818-L1860</a> <a href="#fr-u1-1">↩</a></p> |
| </li> |
| <li id="footnote-res1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_ty_utils/src/instance.rs#L351">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_ty_utils/src/instance.rs#L351</a> <a href="#fr-res1-1">↩</a> <a href="#fr-res1-2">↩2</a></p> |
| </li> |
| <li id="footnote-res2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_ty_utils/src/instance.rs#L341-L349">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_ty_utils/src/instance.rs#L341-L349</a> <a href="#fr-res2-1">↩</a></p> |
| </li> |
| <li id="footnote-res3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_ty_utils/src/instance.rs#L312-L326">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_ty_utils/src/instance.rs#L312-L326</a> <a href="#fr-res3-1">↩</a></p> |
| </li> |
| <li id="footnote-i1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_middle/src/ty/instance.rs#L129-L134">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_middle/src/ty/instance.rs#L129-L134</a> <a href="#fr-i1-1">↩</a></p> |
| </li> |
| <li id="footnote-i2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_middle/src/ty/instance.rs#L136-L141">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_middle/src/ty/instance.rs#L136-L141</a> <a href="#fr-i2-1">↩</a></p> |
| </li> |
| <li id="footnote-i3"> |
| <p><a href="https://github.com/rust-lang/rust/blob/07cbbdd69363da97075650e9be24b78af0bcdd23/compiler/rustc_middle/src/ty/instance.rs#L841">https://github.com/rust-lang/rust/blob/07cbbdd69363da97075650e9be24b78af0bcdd23/compiler/rustc_middle/src/ty/instance.rs#L841</a> <a href="#fr-i3-1">↩</a></p> |
| </li> |
| <li id="footnote-bck1"> |
| <p><a href="https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_borrowck/src/universal_regions.rs#L110-L115">https://github.com/rust-lang/rust/blob/705cfe0e966399e061d64dd3661bfbc57553ed87/compiler/rustc_borrowck/src/universal_regions.rs#L110-L115</a> <a href="#fr-bck1-1">↩</a></p> |
| </li> |
| <li id="footnote-bck2"> |
| <p><a href="https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_borrowck/src/universal_regions.rs#L743-L790">https://github.com/rust-lang/rust/blob/7c7bb7dc017545db732f5cffec684bbaeae0a9a0/compiler/rustc_borrowck/src/universal_regions.rs#L743-L790</a> <a href="#fr-bck2-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="from-mir-to-binaries"><a class="header" href="#from-mir-to-binaries">From MIR to binaries</a></h1> |
| <p>All of the preceding chapters of this guide have one thing in common: |
| we never generated any executable machine code at all! |
| With this chapter, all of that changes.</p> |
| <p>So far, |
| we've shown how the compiler can take raw source code in text format |
| and transform it into <a href="./mir/index.html">MIR</a>. |
| We have also shown how the compiler does various |
| analyses on the code to detect things like type or lifetime errors. |
| Now, we will finally take the MIR and produce some executable machine code.</p> |
| <blockquote> |
| <p>NOTE: This part of a compiler is often called the <em>backend</em>. |
| The term is a bit overloaded because in the compiler source, |
| it usually refers to the "codegen backend" (i.e. LLVM, Cranelift, or GCC). |
| Usually, when you see the word "backend" in this part, |
| we are referring to the "codegen backend".</p> |
| </blockquote> |
| <p>So what do we need to do?</p> |
| <ol> |
| <li>First, we need to collect the set of things to generate code for. |
| In particular, |
| we need to find out which concrete types to substitute for generic ones, |
| since we need to generate code for the concrete types. |
| Generating code for the concrete types |
| (i.e. emitting a copy of the code for each concrete type) is called <em>monomorphization</em>, |
| so the process of collecting all the concrete types is called <em>monomorphization collection</em>.</li> |
| <li>Next, we need to actually lower the MIR to a codegen IR |
| (usually LLVM IR) for each concrete type we collected.</li> |
| <li>Finally, we need to invoke the codegen backend, |
| which runs a bunch of optimization passes, |
| generates executable code, |
| and links together an executable binary.</li> |
| </ol> |
| <p>The code for codegen is actually a bit complex due to a few factors:</p> |
| <ul> |
| <li>Support for multiple codegen backends (LLVM, Cranelift, and GCC). |
| We try to share as much backend code between them as possible, |
| so a lot of it is generic over the codegen implementation. |
| This means that there are often a lot of layers of abstraction.</li> |
| <li>Codegen happens asynchronously in another thread for performance.</li> |
| <li>The actual codegen is done by a third-party library (either of the 3 backends).</li> |
| </ul> |
| <p>Generally, the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/index.html"><code>rustc_codegen_ssa</code></a> crate contains backend-agnostic code, |
| while the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/index.html"><code>rustc_codegen_llvm</code></a> crate contains code specific to LLVM codegen.</p> |
| <p>At a very high level, the entry point is |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/fn.codegen_crate.html"><code>rustc_codegen_ssa::base::codegen_crate</code></a>. |
| This function starts the process discussed in the rest of this chapter.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mir-optimizations"><a class="header" href="#mir-optimizations">MIR optimizations</a></h1> |
| <p>MIR optimizations are optimizations run on the <a href="mir/../mir/index.html">MIR</a> to produce better MIR |
| before codegen. This is important for two reasons: first, it makes the final |
| generated executable code better, and second, it means that LLVM has less work |
| to do, so compilation is faster. Note that since MIR is generic (not |
| <a href="mir/../appendix/glossary.html#mono">monomorphized</a> yet), these optimizations are particularly |
| effective; we can optimize the generic version, so all of the monomorphizations |
| are cheaper!</p> |
| <p>MIR optimizations run after borrow checking. We run a series of optimization |
| passes over the MIR to improve it. Some passes are required to run on all code, |
| some passes don't actually do optimizations but only check stuff, and some |
| passes are only turned on in <code>release</code> mode.</p> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/fn.optimized_mir.html"><code>optimized_mir</code></a> <a href="mir/../query.html">query</a> is called to produce the optimized MIR |
| for a given <a href="mir/../appendix/glossary.html#def-id"><code>DefId</code></a>. This query makes sure that the borrow checker has |
| run and that some validation has occurred. Then, it <a href="mir/../mir/passes.html#stealing">steals</a> the MIR, |
| optimizes it, and returns the improved MIR.</p> |
| <h2 id="quickstart-for-adding-a-new-optimization"><a class="header" href="#quickstart-for-adding-a-new-optimization">Quickstart for adding a new optimization</a></h2> |
| <ol> |
| <li> |
| <p>Make a Rust source file in <code>tests/mir-opt</code> that shows the code you want to |
| optimize. This should be kept simple, so avoid <code>println!</code> or other formatting |
| code if it's not necessary for the optimization. The reason for this is that |
| <code>println!</code>, <code>format!</code>, etc. generate a lot of MIR that can make it harder to |
| understand what the optimization does to the test.</p> |
| </li> |
| <li> |
| <p>Run <code>./x test --bless tests/mir-opt/<your-test>.rs</code> to generate a MIR |
| dump. Read <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/mir-opt/README.md">this README</a> for instructions on how to dump |
| things.</p> |
| </li> |
| <li> |
| <p>Commit the current working directory state. The reason you should commit the |
| test output before you implement the optimization is so that you (and your |
| reviewers) can see a before/after diff of what the optimization changed.</p> |
| </li> |
| <li> |
| <p>Implement a new optimization in <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_mir_transform/src"><code>compiler/rustc_mir_transform/src</code></a>. |
| The fastest and easiest way to do this is to</p> |
| <ol> |
| <li>pick a small optimization (such as <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_mir_transform/src/remove_storage_markers.rs"><code>remove_storage_markers</code></a>) and copy it |
| to a new file,</li> |
| <li>add your optimization to one of the lists in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/fn.run_optimization_passes.html"><code>run_optimization_passes()</code></a> function,</li> |
| <li>and then start modifying the copied optimization.</li> |
| </ol> |
| </li> |
| <li> |
| <p>Rerun <code>./x test --bless tests/mir-opt/<your-test>.rs</code> to regenerate the |
| MIR dumps. Look at the diffs to see if they are what you expect.</p> |
| </li> |
| <li> |
| <p>Run <code>./x test tests/ui</code> to see if your optimization broke anything.</p> |
| </li> |
| <li> |
| <p>If there are issues with your optimization, experiment with it a bit and |
| repeat steps 5 and 6.</p> |
| </li> |
| <li> |
| <p>Commit and open a PR. You can do this at any point, even if things aren't |
| working yet, so that you can ask for feedback on the PR. Open a "WIP" PR |
| (just prefix your PR title with <code>[WIP]</code> or otherwise note that it is a |
| work in progress) in that case.</p> |
| <p>Make sure to commit the blessed test output as well! It's necessary for CI to |
| pass and it's very helpful to reviewers.</p> |
| </li> |
| </ol> |
| <p>If you have any questions along the way, feel free to ask in |
| <code>#t-compiler/wg-mir-opt</code> on Zulip.</p> |
| <h2 id="defining-optimization-passes"><a class="header" href="#defining-optimization-passes">Defining optimization passes</a></h2> |
| <p>The list of passes run and the order in which they are run is defined by the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/fn.run_optimization_passes.html"><code>run_optimization_passes</code></a> function. It contains an array of passes to |
| run. Each pass in the array is a struct that implements the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/pass_manager/trait.MirPass.html"><code>MirPass</code></a> trait. |
| The array is an array of <code>&dyn MirPass</code> trait objects. Typically, a pass is |
| implemented in its own module of the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/index.html"><code>rustc_mir_transform</code></a> crate.</p> |
| <p>Some examples of passes are:</p> |
| <ul> |
| <li><code>CleanupPostBorrowck</code>: Remove some of the info that is only needed for |
| analyses, rather than codegen.</li> |
| <li><code>ConstProp</code>: Does <a href="https://en.wikipedia.org/wiki/Constant_folding#Constant_propagation">constant propagation</a>.</li> |
| </ul> |
| <p>You can see the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/pass_manager/trait.MirPass.html#implementors">"Implementors" section of the <code>MirPass</code> rustdocs</a> for more examples.</p> |
| <h2 id="mir-optimization-levels"><a class="header" href="#mir-optimization-levels">MIR optimization levels</a></h2> |
| <p>MIR optimizations can come in various levels of readiness. Experimental |
| optimizations may cause miscompilations, or slow down compile times. |
| These passes are still included in nightly builds to gather feedback and make it easier to modify |
| the pass. To enable working with slow or otherwise experimental optimization passes, |
| you can specify the <code>-Z mir-opt-level</code> debug flag. You can find the |
| definitions of the levels in the <a href="https://github.com/rust-lang/compiler-team/issues/319">compiler MCP</a>. If you are developing a MIR pass and |
| want to query whether your optimization pass should run, you can check the |
| current level using <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/config/struct.UnstableOptions.html#structfield.mir_opt_level"><code>tcx.sess.opts.unstable_opts.mir_opt_level</code></a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="mir-debugging"><a class="header" href="#mir-debugging">MIR Debugging</a></h1> |
| <p>The <code>-Z dump-mir</code> flag can be used to dump a text representation of the MIR. |
| The following optional flags, used in combination with <code>-Z dump-mir</code>, enable |
| additional output formats, including:</p> |
| <ul> |
| <li><code>-Z dump-mir-graphviz</code> - dumps a <code>.dot</code> file that represents MIR as a |
| control-flow graph</li> |
| <li><code>-Z dump-mir-dataflow</code> - dumps a <code>.dot</code> file showing the <a href="mir/./dataflow.html#graphviz-diagrams">dataflow state</a> at |
| each point in the control-flow graph</li> |
| </ul> |
| <p><code>-Z dump-mir=F</code> is a handy compiler option that will let you view the MIR for |
| each function at each stage of compilation. <code>-Z dump-mir</code> takes a <strong>filter</strong> <code>F</code> |
| which allows you to control which functions and which passes you are |
| interested in. For example:</p> |
| <pre><code class="language-bash">> rustc -Z dump-mir=foo ... |
| </code></pre> |
| <p>This will dump the MIR for any function whose name contains <code>foo</code>; it |
| will dump the MIR both before and after every pass. Those files will |
| be created in the <code>mir_dump</code> directory. There will likely be quite a |
| lot of them!</p> |
| <pre><code class="language-bash">> cat > foo.rs |
| fn main() { |
| println!("Hello, world!"); |
| } |
| ^D |
| > rustc -Z dump-mir=main foo.rs |
| > ls mir_dump/* | wc -l |
| 161 |
| </code></pre> |
| <p>The files have names like <code>rustc.main.000-000.CleanEndRegions.after.mir</code>. These |
| names have a number of parts:</p> |
| <pre><code class="language-text">rustc.main.000-000.CleanEndRegions.after.mir |
| ---- --- --- --------------- ----- either before or after |
| | | | name of the pass |
| | | index of dump within the pass (usually 0, but some passes dump intermediate states) |
| | index of the pass |
| def-path to the function etc being dumped |
| </code></pre> |
| <p>You can also make more selective filters. For example, <code>main & CleanEndRegions</code> |
| will select for things that reference <em>both</em> <code>main</code> and the pass |
| <code>CleanEndRegions</code>:</p> |
| <pre><code class="language-bash">> rustc -Z dump-mir='main & CleanEndRegions' foo.rs |
| > ls mir_dump |
| rustc.main.000-000.CleanEndRegions.after.mir rustc.main.000-000.CleanEndRegions.before.mir |
| </code></pre> |
| <!--- TODO: Change NoLandingPads. [#1232](https://github.com/rust-lang/rustc-dev-guide/issues/1232) --> |
| <p>Filters can also have <code>|</code> parts to combine multiple sets of |
| <code>&</code>-filters. For example <code>main & CleanEndRegions | main & NoLandingPads</code> will select <em>either</em> <code>main</code> and <code>CleanEndRegions</code> <em>or</em> |
| <code>main</code> and <code>NoLandingPads</code>:</p> |
| <pre><code class="language-bash">> rustc -Z dump-mir='main & CleanEndRegions | main & NoLandingPads' foo.rs |
| > ls mir_dump |
| rustc.main-promoted[0].002-000.NoLandingPads.after.mir |
| rustc.main-promoted[0].002-000.NoLandingPads.before.mir |
| rustc.main-promoted[0].002-006.NoLandingPads.after.mir |
| rustc.main-promoted[0].002-006.NoLandingPads.before.mir |
| rustc.main-promoted[1].002-000.NoLandingPads.after.mir |
| rustc.main-promoted[1].002-000.NoLandingPads.before.mir |
| rustc.main-promoted[1].002-006.NoLandingPads.after.mir |
| rustc.main-promoted[1].002-006.NoLandingPads.before.mir |
| rustc.main.000-000.CleanEndRegions.after.mir |
| rustc.main.000-000.CleanEndRegions.before.mir |
| rustc.main.002-000.NoLandingPads.after.mir |
| rustc.main.002-000.NoLandingPads.before.mir |
| rustc.main.002-006.NoLandingPads.after.mir |
| rustc.main.002-006.NoLandingPads.before.mir |
| </code></pre> |
| <p>(Here, the <code>main-promoted[0]</code> files refer to the MIR for "promoted constants" |
| that appeared within the <code>main</code> function.)</p> |
| <p>The <code>-Z unpretty=mir-cfg</code> flag can be used to create a graphviz MIR |
| control-flow diagram for the whole crate:</p> |
| <p><img src="mir/mir_cfg.svg" alt="A control-flow diagram" /></p> |
| <p>TODO: anything else?</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="constant-evaluation"><a class="header" href="#constant-evaluation">Constant Evaluation</a></h1> |
| <p>Constant evaluation is the process of computing values at compile time. For a |
| specific item (constant/static/array length) this happens after the MIR for the |
| item is borrow-checked and optimized. In many cases trying to const evaluate an |
| item will trigger the computation of its MIR for the first time.</p> |
| <p>Prominent examples are:</p> |
| <ul> |
| <li>The initializer of a <code>static</code></li> |
| <li>Array length |
| <ul> |
| <li>needs to be known to reserve stack or heap space</li> |
| </ul> |
| </li> |
| <li>Enum variant discriminants |
| <ul> |
| <li>needs to be known to prevent two variants from having the same |
| discriminant</li> |
| </ul> |
| </li> |
| <li>Patterns |
| <ul> |
| <li>need to be known to check for overlapping patterns</li> |
| </ul> |
| </li> |
| </ul> |
| <p>Additionally constant evaluation can be used to reduce the workload or binary |
| size at runtime by precomputing complex operations at compile time and only |
| storing the result.</p> |
| <p>All uses of constant evaluation can either be categorized as "influencing the type system" |
| (array lengths, enum variant discriminants, const generic parameters), or as solely being |
| done to precompute expressions to be used at runtime.</p> |
| <p>Constant evaluation can be done by calling the <code>const_eval_*</code> functions of <code>TyCtxt</code>. |
| They're the wrappers of the <code>const_eval</code> query.</p> |
| <ul> |
| <li><code>const_eval_global_id_for_typeck</code> evaluates a constant to a valtree, |
| so the result value can be further inspected by the compiler.</li> |
| <li><code>const_eval_global_id</code> evaluate a constant to an "opaque blob" containing its final value; |
| this is only useful for codegen backends and the CTFE evaluator engine itself.</li> |
| <li><code>eval_static_initializer</code> specifically computes the initial values of a static. |
| Statics are special; all other functions do not represent statics correctly |
| and have thus assertions preventing their use on statics.</li> |
| </ul> |
| <p>The <code>const_eval_*</code> functions use a <a href="./typing_parameter_envs.html"><code>ParamEnv</code></a> of environment |
| in which the constant is evaluated (e.g. the function within which the constant is used) |
| and a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/struct.GlobalId.html"><code>GlobalId</code></a>. The <code>GlobalId</code> is made up of an <code>Instance</code> referring to a constant |
| or static or of an <code>Instance</code> of a function and an index into the function's <code>Promoted</code> table.</p> |
| <p>Constant evaluation returns an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/error/type.EvalToValTreeResult.html"><code>EvalToValTreeResult</code></a> for type system constants |
| or <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/error/type.EvalToConstValueResult.html"><code>EvalToConstValueResult</code></a> with either the error, or a representation of the |
| evaluated constant: a <a href="mir/index.html#valtrees">valtree</a> or a <a href="mir/index.html#mir-constant-values">MIR constant |
| value</a>, respectively.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="interpreter"><a class="header" href="#interpreter">Interpreter</a></h1> |
| <p>The interpreter is a virtual machine for executing MIR without compiling to |
| machine code. It is usually invoked via <code>tcx.const_eval_*</code> functions. The |
| interpreter is shared between the compiler (for compile-time function |
| evaluation, CTFE) and the tool <a href="https://github.com/rust-lang/miri/">Miri</a>, which |
| uses the same virtual machine to detect Undefined Behavior in (unsafe) Rust |
| code.</p> |
| <p>If you start out with a constant:</p> |
| <pre><pre class="playground"><code class="language-rust"><span class="boring">#![allow(unused)] |
| </span><span class="boring">fn main() { |
| </span>const FOO: usize = 1 << 12; |
| <span class="boring">}</span></code></pre></pre> |
| <p>rustc doesn't actually invoke anything until the constant is either used or |
| placed into metadata.</p> |
| <p>Once you have a use-site like:</p> |
| <pre><code class="language-rust ignore">type Foo = [u8; FOO - 42];</code></pre> |
| <p>The compiler needs to figure out the length of the array before being able to |
| create items that use the type (locals, constants, function arguments, ...).</p> |
| <p>To obtain the (in this case empty) parameter environment, one can call |
| <code>let param_env = tcx.param_env(length_def_id);</code>. The <code>GlobalId</code> needed is</p> |
| <pre><code class="language-rust ignore">let gid = GlobalId { |
| promoted: None, |
| instance: Instance::mono(length_def_id), |
| };</code></pre> |
| <p>Invoking <code>tcx.const_eval(param_env.and(gid))</code> will now trigger the creation of |
| the MIR of the array length expression. The MIR will look something like this:</p> |
| <pre><code class="language-mir">Foo::{{constant}}#0: usize = { |
| let mut _0: usize; |
| let mut _1: (usize, bool); |
| |
| bb0: { |
| _1 = CheckedSub(const FOO, const 42usize); |
| assert(!move (_1.1: bool), "attempt to subtract with overflow") -> bb1; |
| } |
| |
| bb1: { |
| _0 = move (_1.0: usize); |
| return; |
| } |
| } |
| </code></pre> |
| <p>Before the evaluation, a virtual memory location (in this case essentially a |
| <code>vec![u8; 4]</code> or <code>vec![u8; 8]</code>) is created for storing the evaluation result.</p> |
| <p>At the start of the evaluation, <code>_0</code> and <code>_1</code> are |
| <code>Operand::Immediate(Immediate::Scalar(ScalarMaybeUndef::Undef))</code>. This is quite |
| a mouthful: <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/operand/enum.Operand.html"><code>Operand</code></a> can represent either data stored somewhere in the |
| <a href="const-eval/interpret.html#memory">interpreter memory</a> (<code>Operand::Indirect</code>), or (as an optimization) |
| immediate data stored in-line. And <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/enum.Immediate.html"><code>Immediate</code></a> can either be a single |
| (potentially uninitialized) <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/enum.Scalar.html">scalar value</a> (integer or thin pointer), |
| or a pair of two of them. In our case, the single scalar value is <em>not</em> (yet) |
| initialized.</p> |
| <p>When the initialization of <code>_1</code> is invoked, the value of the <code>FOO</code> constant is |
| required, and triggers another call to <code>tcx.const_eval_*</code>, which will not be shown |
| here. If the evaluation of FOO is successful, <code>42</code> will be subtracted from its |
| value <code>4096</code> and the result stored in <code>_1</code> as |
| <code>Operand::Immediate(Immediate::ScalarPair(Scalar::Raw { data: 4054, .. }, Scalar::Raw { data: 0, .. })</code>. The first part of the pair is the computed value, |
| the second part is a bool that's true if an overflow happened. A <code>Scalar::Raw</code> |
| also stores the size (in bytes) of this scalar value; we are eliding that here.</p> |
| <p>The next statement asserts that said boolean is <code>0</code>. In case the assertion |
| fails, its error message is used for reporting a compile-time error.</p> |
| <p>Since it does not fail, <code>Operand::Immediate(Immediate::Scalar(Scalar::Raw { data: 4054, .. }))</code> is stored in the virtual memory it was allocated before the |
| evaluation. <code>_0</code> always refers to that location directly.</p> |
| <p>After the evaluation is done, the return value is converted from <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/operand/enum.Operand.html"><code>Operand</code></a> to |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/consts/enum.ConstValue.html"><code>ConstValue</code></a> by <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/const_eval/eval_queries/fn.op_to_const.html"><code>op_to_const</code></a>: the former representation is geared towards |
| what is needed <em>during</em> const evaluation, while <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/consts/enum.ConstValue.html"><code>ConstValue</code></a> is shaped by the |
| needs of the remaining parts of the compiler that consume the results of const |
| evaluation. As part of this conversion, for types with scalar values, even if |
| the resulting <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/operand/enum.Operand.html"><code>Operand</code></a> is <code>Indirect</code>, it will return an immediate |
| <code>ConstValue::Scalar(computed_value)</code> (instead of the usual <code>ConstValue::ByRef</code>). |
| This makes using the result much more efficient and also more convenient, as no |
| further queries need to be executed in order to get at something as simple as a |
| <code>usize</code>.</p> |
| <p>Future evaluations of the same constants will not actually invoke |
| the interpreter, but just use the cached result.</p> |
| <h2 id="datastructures"><a class="header" href="#datastructures">Datastructures</a></h2> |
| <p>The interpreter's outside-facing datastructures can be found in |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_middle/src/mir/interpret">rustc_middle/src/mir/interpret</a>. |
| This is mainly the error enum and the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/consts/enum.ConstValue.html"><code>ConstValue</code></a> and <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/enum.Scalar.html"><code>Scalar</code></a> types. A |
| <code>ConstValue</code> can be either <code>Scalar</code> (a single <code>Scalar</code>, i.e., integer or thin |
| pointer), <code>Slice</code> (to represent byte slices and strings, as needed for pattern |
| matching) or <code>ByRef</code>, which is used for anything else and refers to a virtual |
| allocation. These allocations can be accessed via the methods on |
| <code>tcx.interpret_interner</code>. A <code>Scalar</code> is either some <code>Raw</code> integer or a pointer; |
| see <a href="const-eval/interpret.html#memory">the next section</a> for more on that.</p> |
| <p>If you are expecting a numeric result, you can use <code>eval_usize</code> (panics on |
| anything that can't be represented as a <code>u64</code>) or <code>try_eval_usize</code> which results |
| in an <code>Option<u64></code> yielding the <code>Scalar</code> if possible.</p> |
| <h2 id="memory"><a class="header" href="#memory">Memory</a></h2> |
| <p>To support any kind of pointers, the interpreter needs to have a "virtual memory" that the |
| pointers can point to. This is implemented in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.Memory.html"><code>Memory</code></a> type. In the |
| simplest model, every global variable, stack variable and every dynamic |
| allocation corresponds to an <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/struct.Allocation.html"><code>Allocation</code></a> in that memory. (Actually using an |
| allocation for every MIR stack variable would be very inefficient; that's why we |
| have <code>Operand::Immediate</code> for stack variables that are both small and never have |
| their address taken. But that is purely an optimization.)</p> |
| <p>Such an <code>Allocation</code> is basically just a sequence of <code>u8</code> storing the value of |
| each byte in this allocation. (Plus some extra data, see below.) Every |
| <code>Allocation</code> has a globally unique <code>AllocId</code> assigned in <code>Memory</code>. With that, a |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/struct.Pointer.html"><code>Pointer</code></a> consists of a pair of an <code>AllocId</code> (indicating the allocation) and |
| an offset into the allocation (indicating which byte of the allocation the |
| pointer points to). It may seem odd that a <code>Pointer</code> is not just an integer |
| address, but remember that during const evaluation, we cannot know at which |
| actual integer address the allocation will end up -- so we use <code>AllocId</code> as |
| symbolic base addresses, which means we need a separate offset. (As an aside, |
| it turns out that pointers at run-time are |
| <a href="https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#pointer-provenance">more than just integers, too</a>.)</p> |
| <p>These allocations exist so that references and raw pointers have something to |
| point to. There is no global linear heap in which things are allocated, but each |
| allocation (be it for a local variable, a static or a (future) heap allocation) |
| gets its own little memory with exactly the required size. So if you have a |
| pointer to an allocation for a local variable <code>a</code>, there is no possible (no |
| matter how unsafe) operation that you can do that would ever change said pointer |
| to a pointer to a different local variable <code>b</code>. |
| Pointer arithmetic on <code>a</code> will only ever change its offset; the <code>AllocId</code> stays the same.</p> |
| <p>This, however, causes a problem when we want to store a <code>Pointer</code> into an |
| <code>Allocation</code>: we cannot turn it into a sequence of <code>u8</code> of the right length! |
| <code>AllocId</code> and offset together are twice as big as a pointer "seems" to be. This |
| is what the <code>relocation</code> field of <code>Allocation</code> is for: the byte offset of the |
| <code>Pointer</code> gets stored as a bunch of <code>u8</code>, while its <code>AllocId</code> gets stored |
| out-of-band. The two are reassembled when the <code>Pointer</code> is read from memory. |
| The other bit of extra data an <code>Allocation</code> needs is <code>undef_mask</code> for keeping |
| track of which of its bytes are initialized.</p> |
| <h3 id="global-memory-and-exotic-allocations"><a class="header" href="#global-memory-and-exotic-allocations">Global memory and exotic allocations</a></h3> |
| <p><code>Memory</code> exists only during evaluation; it gets destroyed when the |
| final value of the constant is computed. In case that constant contains any |
| pointers, those get "interned" and moved to a global "const eval memory" that is |
| part of <code>TyCtxt</code>. These allocations stay around for the remaining computation |
| and get serialized into the final output (so that dependent crates can use |
| them).</p> |
| <p>Moreover, to also support function pointers, the global memory in <code>TyCtxt</code> can |
| also contain "virtual allocations": instead of an <code>Allocation</code>, these contain an |
| <code>Instance</code>. That allows a <code>Pointer</code> to point to either normal data or a |
| function, which is needed to be able to evaluate casts from function pointers to |
| raw pointers.</p> |
| <p>Finally, the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/enum.GlobalAlloc.html"><code>GlobalAlloc</code></a> type used in the global memory also contains a |
| variant <code>Static</code> that points to a particular <code>const</code> or <code>static</code> item. This is |
| needed to support circular statics, where we need to have a <code>Pointer</code> to a |
| <code>static</code> for which we cannot yet have an <code>Allocation</code> as we do not know the |
| bytes of its value.</p> |
| <h3 id="pointer-values-vs-pointer-types"><a class="header" href="#pointer-values-vs-pointer-types">Pointer values vs Pointer types</a></h3> |
| <p>One common cause of confusion in the interpreter is that being a pointer <em>value</em> and having |
| a pointer <em>type</em> are entirely independent properties. By "pointer value", we |
| refer to a <code>Scalar::Ptr</code> containing a <code>Pointer</code> and thus pointing somewhere into |
| the interpreter's virtual memory. This is in contrast to <code>Scalar::Raw</code>, which is just some |
| concrete integer.</p> |
| <p>However, a variable of pointer or reference <em>type</em>, such as <code>*const T</code> or <code>&T</code>, |
| does not have to have a pointer <em>value</em>: it could be obtained by casting or |
| transmuting an integer to a pointer. |
| And similarly, when casting or transmuting a reference to some |
| actual allocation to an integer, we end up with a pointer <em>value</em> |
| (<code>Scalar::Ptr</code>) at integer <em>type</em> (<code>usize</code>). This is a problem because we |
| cannot meaningfully perform integer operations such as division on pointer |
| values.</p> |
| <h2 id="interpretation"><a class="header" href="#interpretation">Interpretation</a></h2> |
| <p>Although the main entry point to constant evaluation is the <code>tcx.const_eval_*</code> |
| functions, there are additional functions in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/index.html">rustc_const_eval/src/const_eval</a> |
| that allow accessing the fields of a <code>ConstValue</code> (<code>ByRef</code> or otherwise). You should |
| never have to access an <code>Allocation</code> directly except for translating it to the |
| compilation target (at the moment just LLVM).</p> |
| <p>The interpreter starts by creating a virtual stack frame for the current constant that is |
| being evaluated. There's essentially no difference between a constant and a |
| function with no arguments, except that constants do not allow local (named) |
| variables at the time of writing this guide.</p> |
| <p>A stack frame is defined by the <code>Frame</code> type in |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_const_eval/src/interpret/eval_context.rs">rustc_const_eval/src/interpret/eval_context.rs</a> |
| and contains all the local |
| variables memory (<code>None</code> at the start of evaluation). Each frame refers to the |
| evaluation of either the root constant or subsequent calls to <code>const fn</code>. The |
| evaluation of another constant simply calls <code>tcx.const_eval_*</code>, which produce an |
| entirely new and independent stack frame.</p> |
| <p>The frames are just a <code>Vec<Frame></code>, there's no way to actually refer to a |
| <code>Frame</code>'s memory even if horrible shenanigans are done via unsafe code. The only |
| memory that can be referred to are <code>Allocation</code>s.</p> |
| <p>The interpreter now calls the <code>step</code> method (in |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_const_eval/src/interpret/step.rs">rustc_const_eval/src/interpret/step.rs</a> |
| ) until it either returns an error or has no further statements to execute. Each |
| statement will now initialize or modify the locals or the virtual memory |
| referred to by a local. This might require evaluating other constants or |
| statics, which just recursively invokes <code>tcx.const_eval_*</code>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="monomorphization"><a class="header" href="#monomorphization">Monomorphization</a></h1> |
| <p>As you probably know, Rust has a very expressive type system that has extensive |
| support for generic types. But of course, assembly is not generic, so we need |
| to figure out the concrete types of all the generics before the code can |
| execute.</p> |
| <p>Different languages handle this problem differently. For example, in some |
| languages, such as Java, we may not know the most precise type of value until |
| runtime. In the case of Java, this is ok because (almost) all variables are |
| reference values anyway (i.e. pointers to a heap allocated object). This |
| flexibility comes at the cost of performance, since all accesses to an object |
| must dereference a pointer.</p> |
| <p>Rust takes a different approach: it <em>monomorphizes</em> all generic types. This |
| means that compiler stamps out a different copy of the code of a generic |
| function for each concrete type needed. For example, if I use a <code>Vec<u64></code> and |
| a <code>Vec<String></code> in my code, then the generated binary will have two copies of |
| the generated code for <code>Vec</code>: one for <code>Vec<u64></code> and another for <code>Vec<String></code>. |
| The result is fast programs, but it comes at the cost of compile time (creating |
| all those copies can take a while) and binary size (all those copies might take |
| a lot of space).</p> |
| <p>Monomorphization is the first step in the backend of the Rust compiler.</p> |
| <h2 id="collection"><a class="header" href="#collection">Collection</a></h2> |
| <p>First, we need to figure out what concrete types we need for all the generic |
| things in our program. This is called <em>collection</em>, and the code that does this |
| is called the <em>monomorphization collector</em>.</p> |
| <p>Take this example:</p> |
| <pre><pre class="playground"><code class="language-rust">fn banana() { |
| peach::<u64>(); |
| } |
| |
| fn main() { |
| banana(); |
| }</code></pre></pre> |
| <p>The monomorphization collector will give you a list of <code>[main, banana, peach::<u64>]</code>. These are the functions that will have machine code generated |
| for them. Collector will also add things like statics to that list.</p> |
| <p>See <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_monomorphize/collector/index.html">the collector rustdocs</a> for more info.</p> |
| <p>The monomorphization collector is run just before MIR lowering and codegen. |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/fn.codegen_crate.html"><code>rustc_codegen_ssa::base::codegen_crate</code></a> calls the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_monomorphize/partitioning/fn.collect_and_partition_mono_items.html"><code>collect_and_partition_mono_items</code></a> query, which does monomorphization |
| collection and then partitions them into <a href="backend/../appendix/glossary.html#codegen-unit">codegen |
| units</a>.</p> |
| <h2 id="codegen-unit-cgu-partitioning"><a class="header" href="#codegen-unit-cgu-partitioning">Codegen Unit (CGU) partitioning</a></h2> |
| <p>For better incremental build times, the CGU partitioner creates two CGU for each source level |
| modules. One is for "stable" i.e. non-generic code and the other is more volatile code i.e. |
| monomorphized/specialized instances.</p> |
| <p>For dependencies, consider Crate A and Crate B, such that Crate B depends on Crate A. |
| The following table lists different scenarios for a function in Crate A that might be used by one |
| or more modules in Crate B.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Crate A function</th><th>Behavior</th></tr></thead><tbody> |
| <tr><td>Non-generic function</td><td>Crate A function doesn't appear in any codegen units of Crate B</td></tr> |
| <tr><td>Non-generic <code>#[inline]</code> function</td><td>Crate A function appears within a single CGU of Crate B, and exists even after post-inlining stage</td></tr> |
| <tr><td>Generic function</td><td>Regardless of inlining, all monomorphized (specialized) functions <br> from Crate A appear within a single codegen unit for Crate B. <br> The codegen unit exists even after the post inlining stage.</td></tr> |
| <tr><td>Generic <code>#[inline]</code> function</td><td>- same -</td></tr> |
| </tbody></table> |
| </div> |
| <p>For more details about the partitioner read the module level <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_monomorphize/partitioning/index.html">documentation</a>.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="lowering-mir-to-a-codegen-ir"><a class="header" href="#lowering-mir-to-a-codegen-ir">Lowering MIR to a Codegen IR</a></h1> |
| <p>Now that we have a list of symbols to generate from the collector, we need to |
| generate some sort of codegen IR. In this chapter, we will assume LLVM IR, |
| since that's what rustc usually uses. The actual monomorphization is performed |
| as we go, while we do the translation.</p> |
| <p>Recall that the backend is started by |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/fn.codegen_crate.html"><code>rustc_codegen_ssa::base::codegen_crate</code></a>. Eventually, this reaches |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/fn.codegen_mir.html"><code>rustc_codegen_ssa::mir::codegen_mir</code></a>, which does the lowering from |
| MIR to LLVM IR.</p> |
| <p>The code is split into modules which handle particular MIR primitives:</p> |
| <ul> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/block/index.html"><code>rustc_codegen_ssa::mir::block</code></a> will deal with translating |
| blocks and their terminators. The most complicated and also the most |
| interesting thing this module does is generating code for function calls, |
| including the necessary unwinding handling IR.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/statement/index.html"><code>rustc_codegen_ssa::mir::statement</code></a> translates MIR statements.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/operand/index.html"><code>rustc_codegen_ssa::mir::operand</code></a> translates MIR operands.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/place/index.html"><code>rustc_codegen_ssa::mir::place</code></a> translates MIR place references.</li> |
| <li><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/rvalue/index.html"><code>rustc_codegen_ssa::mir::rvalue</code></a> translates MIR r-values.</li> |
| </ul> |
| <p>Before a function is translated a number of simple and primitive analysis |
| passes will run to help us generate simpler and more efficient LLVM IR. An |
| example of such an analysis pass would be figuring out which variables are |
| SSA-like, so that we can translate them to SSA directly rather than relying on |
| LLVM's <code>mem2reg</code> for those variables. The analysis can be found in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/analyze/index.html"><code>rustc_codegen_ssa::mir::analyze</code></a>.</p> |
| <p>Usually a single MIR basic block will map to a LLVM basic block, with very few |
| exceptions: intrinsic or function calls and less basic MIR statements like |
| <code>assert</code> can result in multiple basic blocks. This is a perfect lede into the |
| non-portable LLVM-specific part of the code generation. Intrinsic generation is |
| fairly easy to understand as it involves very few abstraction levels in between |
| and can be found in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/intrinsic/index.html"><code>rustc_codegen_llvm::intrinsic</code></a>.</p> |
| <p>Everything else will use the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/builder/index.html">builder interface</a>. This is the code that gets |
| called in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/index.html"><code>rustc_codegen_ssa::mir::*</code></a> modules discussed above.</p> |
| <blockquote> |
| <p>TODO: discuss how constants are generated</p> |
| </blockquote> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="code-generation-2"><a class="header" href="#code-generation-2">Code generation</a></h1> |
| <p>Code generation (or "codegen") is the part of the compiler |
| that actually generates an executable binary. |
| Usually, rustc uses LLVM for code generation, |
| but there is also support for <a href="https://github.com/bytecodealliance/wasmtime/tree/main/cranelift">Cranelift</a> and <a href="https://github.com/rust-lang/rustc_codegen_gcc">GCC</a>. |
| The key is that rustc doesn't implement codegen itself. |
| It's worth noting, though, that in the Rust source code, |
| many parts of the backend have <code>codegen</code> in their names |
| (there are no hard boundaries).</p> |
| <blockquote> |
| <p>NOTE: If you are looking for hints on how to debug code generation bugs, |
| please see <a href="backend/./debugging.html">this section of the debugging chapter</a>.</p> |
| </blockquote> |
| <h2 id="what-is-llvm"><a class="header" href="#what-is-llvm">What is LLVM?</a></h2> |
| <p><a href="https://llvm.org">LLVM</a> is "a collection of modular and reusable compiler and |
| toolchain technologies". In particular, the LLVM project contains a pluggable |
| compiler backend (also called "LLVM"), which is used by many compiler projects, |
| including the <code>clang</code> C compiler and our beloved <code>rustc</code>.</p> |
| <p>LLVM takes input in the form of LLVM IR. It is basically assembly code with |
| additional low-level types and annotations added. These annotations are helpful |
| for doing optimizations on the LLVM IR and outputted machine code. The end |
| result of all this is (at long last) something executable (e.g. an ELF object, |
| an EXE, or wasm).</p> |
| <p>There are a few benefits to using LLVM:</p> |
| <ul> |
| <li>We don't have to write a whole compiler backend. This reduces implementation |
| and maintenance burden.</li> |
| <li>We benefit from the large suite of advanced optimizations that the LLVM |
| project has been collecting.</li> |
| <li>We can automatically compile Rust to any of the platforms for which LLVM has |
| support. For example, as soon as LLVM added support for wasm, voila! rustc, |
| clang, and a bunch of other languages were able to compile to wasm! (Well, |
| there was some extra stuff to be done, but we were 90% there anyway).</li> |
| <li>We and other compiler projects benefit from each other. For example, when the |
| <a href="https://meltdownattack.com/">Spectre and Meltdown security vulnerabilities</a> were discovered, |
| only LLVM needed to be patched.</li> |
| </ul> |
| <h2 id="running-llvm-linking-and-metadata-generation"><a class="header" href="#running-llvm-linking-and-metadata-generation">Running LLVM, linking, and metadata generation</a></h2> |
| <p>Once LLVM IR for all of the functions and statics, etc is built, it is time to |
| start running LLVM and its optimization passes. LLVM IR is grouped into |
| "modules". Multiple "modules" can be codegened at the same time to aid in |
| multi-core utilization. These "modules" are what we refer to as <em>codegen |
| units</em>. These units were established way back during monomorphization |
| collection phase.</p> |
| <p>Once LLVM produces objects from these modules, these objects are passed to the |
| linker along with, optionally, the metadata object and an archive or an |
| executable is produced.</p> |
| <p>It is not necessarily the codegen phase described above that runs the |
| optimizations. With certain kinds of LTO, the optimization might happen at the |
| linking time instead. It is also possible for some optimizations to happen |
| before objects are passed on to the linker and some to happen during the |
| linking.</p> |
| <p>This all happens towards the very end of compilation. The code for this can be |
| found in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/back/index.html"><code>rustc_codegen_ssa::back</code></a> and |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/back/index.html"><code>rustc_codegen_llvm::back</code></a>. Sadly, this piece of code is not |
| really well-separated into LLVM-dependent code; the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/index.html"><code>rustc_codegen_ssa</code></a> |
| contains a fair amount of code specific to the LLVM backend.</p> |
| <p>Once these components are done with their work you end up with a number of |
| files in your filesystem corresponding to the outputs you have requested.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="updating-llvm"><a class="header" href="#updating-llvm">Updating LLVM</a></h1> |
| <!-- date-check: Aug 2024 --> |
| <p>Rust supports building against multiple LLVM versions:</p> |
| <ul> |
| <li>Tip-of-tree for the current LLVM development branch is usually supported |
| within a few days. PRs for such fixes are tagged with <code>llvm-main</code>.</li> |
| <li>The latest released major version is always supported.</li> |
| <li>The one or two preceding major versions are usually supported.</li> |
| </ul> |
| <p>By default, Rust uses its own fork in the <a href="https://github.com/rust-lang/llvm-project">rust-lang/llvm-project repository</a>. |
| This fork is based on a <code>release/$N.x</code> branch of the upstream project, where |
| <code>$N</code> is either the latest released major version, or the current major version |
| in release candidate phase. The fork is never based on the <code>main</code> development |
| branch.</p> |
| <p>Our LLVM fork only accepts:</p> |
| <ul> |
| <li>Backports of changes that have already landed upstream.</li> |
| <li>Workarounds for build issues affecting our CI environment.</li> |
| </ul> |
| <p>With the exception of one grandfathered-in patch for SGX enablement, we do not |
| accept functional patches that have not been upstreamed first.</p> |
| <p>There are three types of LLVM updates, with different procedures:</p> |
| <ul> |
| <li>Backports while the current major LLVM version is supported.</li> |
| <li>Backports while the current major LLVM version is no longer supported (or |
| the change is not eligible for upstream backport).</li> |
| <li>Update to a new major LLVM version.</li> |
| </ul> |
| <h2 id="backports-upstream-supported"><a class="header" href="#backports-upstream-supported">Backports (upstream supported)</a></h2> |
| <p>While the current major LLVM version is supported upstream, fixes should be |
| backported upstream first, and the release branch then merged back into the |
| Rust fork.</p> |
| <ol> |
| <li>Make sure the bugfix is in upstream LLVM.</li> |
| <li>If this hasn't happened already, request a backport to the upstream release |
| branch. If you have LLVM commit access, follow the <a href="https://llvm.org/docs/GitHub.html#backporting-fixes-to-the-release-branches">backport process</a>. |
| Otherwise, open an issue requesting the backport. Continue once the |
| backport has been approved and merged.</li> |
| <li>Identify the branch that rustc is currently using. The <code>src/llvm-project</code> |
| submodule is always pinned to a branch of the |
| <a href="https://github.com/rust-lang/llvm-project">rust-lang/llvm-project repository</a>.</li> |
| <li>Fork the rust-lang/llvm-project repository.</li> |
| <li>Check out the appropriate branch (typically named <code>rustc/a.b-yyyy-mm-dd</code>).</li> |
| <li>Add a remote for the upstream repository using |
| <code>git remote add upstream https://github.com/llvm/llvm-project.git</code> and |
| fetch it using <code>git fetch upstream</code>.</li> |
| <li>Merge the <code>upstream/release/$N.x</code> branch.</li> |
| <li>Push this branch to your fork.</li> |
| <li>Send a Pull Request to rust-lang/llvm-project to the same branch as before. |
| Be sure to reference the Rust and/or LLVM issue that you're fixing in the PR |
| description.</li> |
| <li>Wait for the PR to be merged.</li> |
| <li>Send a PR to rust-lang/rust updating the <code>src/llvm-project</code> submodule with |
| your bugfix. This can be done locally with <code>git submodule update --remote src/llvm-project</code> typically.</li> |
| <li>Wait for PR to be merged.</li> |
| </ol> |
| <p>An example PR: |
| <a href="https://github.com/rust-lang/rust/pull/59089">#59089</a></p> |
| <h2 id="backports-upstream-not-supported"><a class="header" href="#backports-upstream-not-supported">Backports (upstream not supported)</a></h2> |
| <p>Upstream LLVM releases are only supported for two to three months after the |
| GA release. Once upstream backports are no longer accepted, changes should be |
| cherry-picked directly to our fork.</p> |
| <ol> |
| <li>Make sure the bugfix is in upstream LLVM.</li> |
| <li>Identify the branch that rustc is currently using. The <code>src/llvm-project</code> |
| submodule is always pinned to a branch of the |
| <a href="https://github.com/rust-lang/llvm-project">rust-lang/llvm-project repository</a>.</li> |
| <li>Fork the rust-lang/llvm-project repository.</li> |
| <li>Check out the appropriate branch (typically named <code>rustc/a.b-yyyy-mm-dd</code>).</li> |
| <li>Add a remote for the upstream repository using |
| <code>git remote add upstream https://github.com/llvm/llvm-project.git</code> and |
| fetch it using <code>git fetch upstream</code>.</li> |
| <li>Cherry-pick the relevant commit(s) using <code>git cherry-pick -x</code>.</li> |
| <li>Push this branch to your fork.</li> |
| <li>Send a Pull Request to rust-lang/llvm-project to the same branch as before. |
| Be sure to reference the Rust and/or LLVM issue that you're fixing in the PR |
| description.</li> |
| <li>Wait for the PR to be merged.</li> |
| <li>Send a PR to rust-lang/rust updating the <code>src/llvm-project</code> submodule with |
| your bugfix. This can be done locally with <code>git submodule update --remote src/llvm-project</code> typically.</li> |
| <li>Wait for PR to be merged.</li> |
| </ol> |
| <p>An example PR: |
| <a href="https://github.com/rust-lang/rust/pull/59089">#59089</a></p> |
| <h2 id="new-llvm-release-updates"><a class="header" href="#new-llvm-release-updates">New LLVM Release Updates</a></h2> |
| <!-- date-check: Jul 2023 --> |
| <p>Unlike bugfixes, |
| updating to a new release of LLVM typically requires a lot more work. |
| This is where we can't reasonably cherry-pick commits backwards, |
| so we need to do a full update. |
| There's a lot of stuff to do here, |
| so let's go through each in detail.</p> |
| <ol> |
| <li> |
| <p>LLVM announces that its latest release version has branched. |
| This will show up as a branch in the <a href="https://github.com/llvm/llvm-project">llvm/llvm-project repository</a>, |
| typically named <code>release/$N.x</code>, |
| where <code>$N</code> is the version of LLVM that's being released.</p> |
| </li> |
| <li> |
| <p>Create a new branch in the <a href="https://github.com/rust-lang/llvm-project">rust-lang/llvm-project repository</a> |
| from this <code>release/$N.x</code> branch, |
| and name it <code>rustc/a.b-yyyy-mm-dd</code>, |
| where <code>a.b</code> is the current version number of LLVM in-tree |
| at the time of the branch, |
| and the remaining part is the current date.</p> |
| </li> |
| <li> |
| <p>Apply Rust-specific patches to the llvm-project repository. |
| All features and bugfixes are upstream, |
| but there's often some weird build-related patches |
| that don't make sense to upstream. |
| These patches are typically the latest patches in the |
| rust-lang/llvm-project branch that rustc is currently using.</p> |
| </li> |
| <li> |
| <p>Build the new LLVM in the <code>rust</code> repository. |
| To do this, |
| you'll want to update the <code>src/llvm-project</code> repository to your branch, |
| and the revision you've created. |
| It's also typically a good idea to update <code>.gitmodules</code> with the new |
| branch name of the LLVM submodule. |
| Make sure you've committed changes to |
| <code>src/llvm-project</code> to ensure submodule updates aren't reverted. |
| Some commands you should execute are:</p> |
| <ul> |
| <li><code>./x build src/llvm-project</code> - test that LLVM still builds</li> |
| <li><code>./x build</code> - build the rest of rustc</li> |
| </ul> |
| <p>You'll likely need to update <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_llvm/llvm-wrapper"><code>llvm-wrapper/*.cpp</code></a> |
| to compile with updated LLVM bindings. |
| Note that you should use <code>#ifdef</code> and such to ensure |
| that the bindings still compile on older LLVM versions.</p> |
| <p>Note that <code>profile = "compiler"</code> and other defaults set by <code>./x setup</code> |
| download LLVM from CI instead of building it from source. |
| You should disable this temporarily to make sure your changes are being used. |
| This is done by having the following setting in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[llvm] |
| download-ci-llvm = false |
| </code></pre> |
| </li> |
| <li> |
| <p>Test for regressions across other platforms. LLVM often has at least one bug |
| for non-tier-1 architectures, so it's good to do some more testing before |
| sending this to bors! If you're low on resources you can send the PR as-is |
| now to bors, though, and it'll get tested anyway.</p> |
| <p>Ideally, build LLVM and test it on a few platforms:</p> |
| <ul> |
| <li>Linux</li> |
| <li>macOS</li> |
| <li>Windows</li> |
| </ul> |
| <p>Afterwards, run some docker containers that CI also does:</p> |
| <ul> |
| <li><code>./src/ci/docker/run.sh wasm32</code></li> |
| <li><code>./src/ci/docker/run.sh arm-android</code></li> |
| <li><code>./src/ci/docker/run.sh dist-various-1</code></li> |
| <li><code>./src/ci/docker/run.sh dist-various-2</code></li> |
| <li><code>./src/ci/docker/run.sh armhf-gnu</code></li> |
| </ul> |
| </li> |
| <li> |
| <p>Prepare a PR to <code>rust-lang/rust</code>. Work with maintainers of |
| <code>rust-lang/llvm-project</code> to get your commit in a branch of that repository, |
| and then you can send a PR to <code>rust-lang/rust</code>. You'll change at least |
| <code>src/llvm-project</code> and will likely also change <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_llvm/llvm-wrapper"><code>llvm-wrapper</code></a> as well.</p> |
| <!-- date-check: mar 2025 --> |
| <blockquote> |
| <p>For prior art, here are some previous LLVM updates:</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/pull/115959">LLVM 17</a></li> |
| <li><a href="https://github.com/rust-lang/rust/pull/120055">LLVM 18</a></li> |
| <li><a href="https://github.com/rust-lang/rust/pull/127513">LLVM 19</a></li> |
| <li><a href="https://github.com/rust-lang/rust/pull/135763">LLVM 20</a></li> |
| </ul> |
| </blockquote> |
| <p>Note that sometimes it's easiest to land <a href="https://github.com/rust-lang/rust/tree/HEAD/compiler/rustc_llvm/llvm-wrapper"><code>llvm-wrapper</code></a> compatibility as a PR |
| before actually updating <code>src/llvm-project</code>. |
| This way, |
| while you're working through LLVM issues, |
| others interested in trying out the new LLVM can benefit from work you've done |
| to update the C++ bindings.</p> |
| </li> |
| <li> |
| <p>Over the next few months, |
| LLVM will continually push commits to its <code>release/a.b</code> branch. |
| We will often want to have those bug fixes as well. |
| The merge process for that is to use <code>git merge</code> itself to merge LLVM's |
| <code>release/a.b</code> branch with the branch created in step 2. |
| This is typically |
| done multiple times when necessary while LLVM's release branch is baking.</p> |
| </li> |
| <li> |
| <p>LLVM then announces the release of version <code>a.b</code>.</p> |
| </li> |
| <li> |
| <p>After LLVM's official release, |
| we follow the process of creating a new branch on the |
| rust-lang/llvm-project repository again, |
| this time with a new date. |
| It is only then that the PR to update Rust to use that version is merged.</p> |
| <p>The commit history of <code>rust-lang/llvm-project</code> |
| should look much cleaner as a <code>git rebase</code> is done, |
| where just a few Rust-specific commits are stacked on top of stock LLVM's release branch.</p> |
| </li> |
| </ol> |
| <h3 id="caveats-and-gotchas"><a class="header" href="#caveats-and-gotchas">Caveats and gotchas</a></h3> |
| <p>Ideally the above instructions are pretty smooth, but here's some caveats to |
| keep in mind while going through them:</p> |
| <ul> |
| <li>LLVM bugs are hard to find, don't hesitate to ask for help! |
| Bisection is definitely your friend here |
| (yes LLVM takes forever to build, yet bisection is still your friend). |
| Note that you can make use of <a href="https://forge.rust-lang.org/infra/docs/dev-desktop.html">Dev Desktops</a>, |
| which is an initiative to provide the contributors with remote access to powerful hardware.</li> |
| <li>If you've got general questions, <a href="https://rust-lang.zulipchat.com/#narrow/stream/187780-t-compiler.2Fwg-llvm">wg-llvm</a> can help you out.</li> |
| <li>Creating branches is a privileged operation on GitHub, so you'll need someone |
| with write access to create the branches for you most likely.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h2 id="debugging-llvm"><a class="header" href="#debugging-llvm">Debugging LLVM</a></h2> |
| <blockquote> |
| <p>NOTE: If you are looking for info about code generation, please see <a href="backend/./codegen.html">this |
| chapter</a> instead.</p> |
| </blockquote> |
| <p>This section is about debugging compiler bugs in code generation (e.g. why the |
| compiler generated some piece of code or crashed in LLVM). LLVM is a big |
| project on its own that probably needs to have its own debugging document (not |
| that I could find one). But here are some tips that are important in a rustc |
| context:</p> |
| <h3 id="minimize-the-example"><a class="header" href="#minimize-the-example">Minimize the example</a></h3> |
| <p>As a general rule, compilers generate lots of information from analyzing code. |
| Thus, a useful first step is usually to find a minimal example. One way to do |
| this is to</p> |
| <ol> |
| <li> |
| <p>create a new crate that reproduces the issue (e.g. adding whatever crate is |
| at fault as a dependency, and using it from there)</p> |
| </li> |
| <li> |
| <p>minimize the crate by removing external dependencies; that is, moving |
| everything relevant to the new crate</p> |
| </li> |
| <li> |
| <p>further minimize the issue by making the code shorter (there are tools that |
| help with this like <code>creduce</code>)</p> |
| </li> |
| </ol> |
| <p>For more discussion on methodology for steps 2 and 3 above, there is an |
| <a href="https://blog.pnkfx.org/blog/2019/11/18/rust-bug-minimization-patterns/">epic blog post</a> from pnkfelix specifically about Rust program minimization.</p> |
| <h3 id="enable-llvm-internal-checks"><a class="header" href="#enable-llvm-internal-checks">Enable LLVM internal checks</a></h3> |
| <p>The official compilers (including nightlies) have LLVM assertions disabled, |
| which means that LLVM assertion failures can show up as compiler crashes (not |
| ICEs but "real" crashes) and other sorts of weird behavior. If you are |
| encountering these, it is a good idea to try using a compiler with LLVM |
| assertions enabled - either an "alt" nightly or a compiler you build yourself |
| by setting <code>[llvm] assertions=true</code> in your bootstrap.toml - and see whether |
| anything turns up.</p> |
| <p>The rustc build process builds the LLVM tools into |
| <code>./build/<host-triple>/llvm/bin</code>. They can be called directly. |
| These tools include:</p> |
| <ul> |
| <li><a href="https://llvm.org/docs/CommandGuide/llc.html"><code>llc</code></a>, which compiles bitcode (<code>.bc</code> files) to executable code; this can be used to |
| replicate LLVM backend bugs.</li> |
| <li><a href="https://llvm.org/docs/CommandGuide/opt.html"><code>opt</code></a>, a bitcode transformer that runs LLVM optimization passes.</li> |
| <li><a href="https://llvm.org/docs/Bugpoint.html"><code>bugpoint</code></a>, which reduces large test cases to small, useful ones.</li> |
| <li>and many others, some of which are referenced in the text below.</li> |
| </ul> |
| <p>By default, the Rust build system does not check for changes to the LLVM source code or |
| its build configuration settings. So, if you need to rebuild the LLVM that is linked |
| into <code>rustc</code>, first delete the file <code>.llvm-stamp</code>, which should be located |
| in <code>build/<host-triple>/llvm/</code>.</p> |
| <p>The default rustc compilation pipeline has multiple codegen units, which is |
| hard to replicate manually and means that LLVM is called multiple times in |
| parallel. If you can get away with it (i.e. if it doesn't make your bug |
| disappear), passing <code>-C codegen-units=1</code> to rustc will make debugging easier.</p> |
| <h3 id="get-your-hands-on-raw-llvm-input"><a class="header" href="#get-your-hands-on-raw-llvm-input">Get your hands on raw LLVM input</a></h3> |
| <p>For rustc to generate LLVM IR, you need to pass the <code>--emit=llvm-ir</code> flag. If |
| you are building via cargo, use the <code>RUSTFLAGS</code> environment variable (e.g. |
| <code>RUSTFLAGS='--emit=llvm-ir'</code>). This causes rustc to spit out LLVM IR into the |
| target directory.</p> |
| <p><code>cargo llvm-ir [options] path</code> spits out the LLVM IR for a particular function |
| at <code>path</code>. (<code>cargo install cargo-asm</code> installs <code>cargo asm</code> and <code>cargo llvm-ir</code>). <code>--build-type=debug</code> emits code for debug builds. There are also |
| other useful options. Also, debug info in LLVM IR can clutter the output a lot: |
| <code>RUSTFLAGS="-C debuginfo=0"</code> is really useful.</p> |
| <p><code>RUSTFLAGS="-C save-temps"</code> outputs LLVM bitcode at |
| different stages during compilation, which is sometimes useful. The output LLVM |
| bitcode will be in <code>.bc</code> files in the compiler's output directory, set via the |
| <code>--out-dir DIR</code> argument to <code>rustc</code>.</p> |
| <ul> |
| <li> |
| <p>If you are hitting an assertion failure or segmentation fault from the LLVM |
| backend when invoking <code>rustc</code> itself, it is a good idea to try passing each |
| of these <code>.bc</code> files to the <code>llc</code> command, and see if you get the same |
| failure. (LLVM developers often prefer a bug reduced to a <code>.bc</code> file over one |
| that uses a Rust crate for its minimized reproduction.)</p> |
| </li> |
| <li> |
| <p>To get human readable versions of the LLVM bitcode, one just needs to convert |
| the bitcode (<code>.bc</code>) files to <code>.ll</code> files using <code>llvm-dis</code>, which should be in |
| the target local compilation of rustc.</p> |
| </li> |
| </ul> |
| <p>Note that rustc emits different IR depending on whether <code>-O</code> is enabled, even |
| without LLVM's optimizations, so if you want to play with the IR rustc emits, |
| you should:</p> |
| <pre><code class="language-bash">$ rustc +local my-file.rs --emit=llvm-ir -O -C no-prepopulate-passes \ |
| -C codegen-units=1 |
| $ OPT=./build/$TRIPLE/llvm/bin/opt |
| $ $OPT -S -O2 < my-file.ll > my |
| </code></pre> |
| <p>If you just want to get the LLVM IR during the LLVM pipeline, to e.g. see which |
| IR causes an optimization-time assertion to fail, or to see when LLVM performs |
| a particular optimization, you can pass the rustc flag <code>-C llvm-args=-print-after-all</code>, and possibly add <code>-C llvm-args='-filter-print-funcs=EXACT_FUNCTION_NAME</code> (e.g. <code>-C llvm-args='-filter-print-funcs=_ZN11collections3str21_$LT$impl$u20$str$GT$\ 7replace17hbe10ea2e7c809b0bE'</code>).</p> |
| <p>That produces a lot of output into standard error, so you'll want to pipe that |
| to some file. Also, if you are using neither <code>-filter-print-funcs</code> nor <code>-C codegen-units=1</code>, then, because the multiple codegen units run in parallel, the |
| printouts will mix together and you won't be able to read anything.</p> |
| <ul> |
| <li> |
| <p>One caveat to the aforementioned methodology: the <code>-print</code> family of options |
| to LLVM only prints the IR unit that the pass runs on (e.g., just a |
| function), and does not include any referenced declarations, globals, |
| metadata, etc. This means you cannot in general feed the output of <code>-print</code> |
| into <code>llc</code> to reproduce a given problem.</p> |
| </li> |
| <li> |
| <p>Within LLVM itself, calling <code>F.getParent()->dump()</code> at the beginning of |
| <code>SafeStackLegacyPass::runOnFunction</code> will dump the whole module, which |
| may provide better basis for reproduction. (However, you |
| should be able to get that same dump from the <code>.bc</code> files dumped by |
| <code>-C save-temps</code>.)</p> |
| </li> |
| </ul> |
| <p>If you want just the IR for a specific function (say, you want to see why it |
| causes an assertion or doesn't optimize correctly), you can use <code>llvm-extract</code>, |
| e.g.</p> |
| <pre><code class="language-bash">$ ./build/$TRIPLE/llvm/bin/llvm-extract \ |
| -func='_ZN11collections3str21_$LT$impl$u20$str$GT$7replace17hbe10ea2e7c809b0bE' \ |
| -S \ |
| < unextracted.ll \ |
| > extracted.ll |
| </code></pre> |
| <h3 id="investigate-llvm-optimization-passes"><a class="header" href="#investigate-llvm-optimization-passes">Investigate LLVM optimization passes</a></h3> |
| <p>If you are seeing incorrect behavior due to an optimization pass, a very handy |
| LLVM option is <code>-opt-bisect-limit</code>, which takes an integer denoting the index |
| value of the highest pass to run. Index values for taken passes are stable |
| from run to run; by coupling this with software that automates bisecting the |
| search space based on the resulting program, an errant pass can be quickly |
| determined. When an <code>-opt-bisect-limit</code> is specified, all runs are displayed |
| to standard error, along with their index and output indicating if the |
| pass was run or skipped. Setting the limit to an index of -1 (e.g., |
| <code>RUSTFLAGS="-C llvm-args=-opt-bisect-limit=-1"</code>) will show all passes and |
| their corresponding index values.</p> |
| <p>If you want to play with the optimization pipeline, you can use the <a href="https://llvm.org/docs/CommandGuide/opt.html"><code>opt</code></a> tool |
| from <code>./build/<host-triple>/llvm/bin/</code> with the LLVM IR emitted by rustc.</p> |
| <p>When investigating the implementation of LLVM itself, you should be |
| aware of its <a href="https://llvm.org/docs/ProgrammersManual.html#the-llvm-debug-macro-and-debug-option">internal debug infrastructure</a>. |
| This is provided in LLVM Debug builds, which you enable for rustc |
| LLVM builds by changing this setting in the bootstrap.toml:</p> |
| <pre><code>[llvm] |
| # Indicates whether the LLVM assertions are enabled or not |
| assertions = true |
| |
| # Indicates whether the LLVM build is a Release or Debug build |
| optimize = false |
| </code></pre> |
| <p>The quick summary is:</p> |
| <ul> |
| <li>Setting <code>assertions=true</code> enables coarse-grain debug messaging. |
| <ul> |
| <li>beyond that, setting <code>optimize=false</code> enables fine-grain debug messaging.</li> |
| </ul> |
| </li> |
| <li><code>LLVM_DEBUG(dbgs() << msg)</code> in LLVM is like <code>debug!(msg)</code> in <code>rustc</code>.</li> |
| <li>The <code>-debug</code> option turns on all messaging; it is like setting the |
| environment variable <code>RUSTC_LOG=debug</code> in <code>rustc</code>.</li> |
| <li>The <code>-debug-only=<pass1>,<pass2></code> variant is more selective; it is like |
| setting the environment variable <code>RUSTC_LOG=path1,path2</code> in <code>rustc</code>.</li> |
| </ul> |
| <h3 id="getting-help-and-asking-questions"><a class="header" href="#getting-help-and-asking-questions">Getting help and asking questions</a></h3> |
| <p>If you have some questions, head over to the <a href="https://rust-lang.zulipchat.com/">rust-lang Zulip</a> and |
| specifically the <code>#t-compiler/wg-llvm</code> channel.</p> |
| <h3 id="compiler-options-to-know-and-love"><a class="header" href="#compiler-options-to-know-and-love">Compiler options to know and love</a></h3> |
| <p>The <code>-C help</code> and <code>-Z help</code> compiler switches will list out a variety |
| of interesting options you may find useful. Here are a few of the most |
| common that pertain to LLVM development (some of them are employed in the |
| tutorial above):</p> |
| <ul> |
| <li>The <code>--emit llvm-ir</code> option emits a <code><filename>.ll</code> file with LLVM IR in textual format |
| <ul> |
| <li>The <code>--emit llvm-bc</code> option emits in bytecode format (<code><filename>.bc</code>)</li> |
| </ul> |
| </li> |
| <li>Passing <code>-C llvm-args=<foo></code> allows passing pretty much all the |
| options that tools like llc and opt would accept; |
| e.g. <code>-C llvm-args=-print-before-all</code> to print IR before every LLVM |
| pass.</li> |
| <li>The <code>-C no-prepopulate-passes</code> will avoid pre-populate the LLVM pass |
| manager with a list of passes. This will allow you to view the LLVM |
| IR that rustc generates, not the LLVM IR after optimizations.</li> |
| <li>The <code>-C passes=val</code> option allows you to supply a space separated list of extra LLVM passes to run</li> |
| <li>The <code>-C save-temps</code> option saves all temporary output files during compilation</li> |
| <li>The <code>-Z print-llvm-passes</code> option will print out LLVM optimization passes being run</li> |
| <li>The <code>-Z time-llvm-passes</code> option measures the time of each LLVM pass</li> |
| <li>The <code>-Z verify-llvm-ir</code> option will verify the LLVM IR for correctness</li> |
| <li>The <code>-Z no-parallel-backend</code> will disable parallel compilation of distinct compilation units</li> |
| <li>The <code>-Z llvm-time-trace</code> option will output a Chrome profiler compatible JSON file |
| which contains details and timings for LLVM passes.</li> |
| <li>The <code>-C llvm-args=-opt-bisect-limit=<index></code> option allows for bisecting LLVM |
| optimizations.</li> |
| </ul> |
| <h3 id="filing-llvm-bug-reports"><a class="header" href="#filing-llvm-bug-reports">Filing LLVM bug reports</a></h3> |
| <p>When filing an LLVM bug report, you will probably want some sort of minimal |
| working example that demonstrates the problem. The Godbolt compiler explorer is |
| really helpful for this.</p> |
| <ol> |
| <li> |
| <p>Once you have some LLVM IR for the problematic code (see above), you can |
| create a minimal working example with Godbolt. Go to |
| <a href="https://llvm.godbolt.org">llvm.godbolt.org</a>.</p> |
| </li> |
| <li> |
| <p>Choose <code>LLVM-IR</code> as programming language.</p> |
| </li> |
| <li> |
| <p>Use <code>llc</code> to compile the IR to a particular target as is:</p> |
| <ul> |
| <li>There are some useful flags: <code>-mattr</code> enables target features, <code>-march=</code> |
| selects the target, <code>-mcpu=</code> selects the CPU, etc.</li> |
| <li>Commands like <code>llc -march=help</code> output all architectures available, which |
| is useful because sometimes the Rust arch names and the LLVM names do not |
| match.</li> |
| <li>If you have compiled rustc yourself somewhere, in the target directory |
| you have binaries for <code>llc</code>, <code>opt</code>, etc.</li> |
| </ul> |
| </li> |
| <li> |
| <p>If you want to optimize the LLVM-IR, you can use <code>opt</code> to see how the LLVM |
| optimizations transform it.</p> |
| </li> |
| <li> |
| <p>Once you have a godbolt link demonstrating the issue, it is pretty easy to |
| fill in an LLVM bug. Just visit their <a href="https://github.com/llvm/llvm-project/issues">github issues page</a>.</p> |
| </li> |
| </ol> |
| <h3 id="porting-bug-fixes-from-llvm"><a class="header" href="#porting-bug-fixes-from-llvm">Porting bug fixes from LLVM</a></h3> |
| <p>Once you've identified the bug as an LLVM bug, you will sometimes |
| find that it has already been reported and fixed in LLVM, but we haven't |
| gotten the fix yet (or perhaps you are familiar enough with LLVM to fix it yourself).</p> |
| <p>In that case, we can sometimes opt to port the fix for the bug |
| directly to our own LLVM fork, so that rustc can use it more easily. |
| Our fork of LLVM is maintained in <a href="https://github.com/rust-lang/llvm-project/">rust-lang/llvm-project</a>. Once |
| you've landed the fix there, you'll also need to land a PR modifying |
| our submodule commits -- ask around on Zulip for help.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="backend-agnostic-codegen"><a class="header" href="#backend-agnostic-codegen">Backend Agnostic Codegen</a></h1> |
| <p><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/index.html"><code>rustc_codegen_ssa</code></a> |
| provides an abstract interface for all backends to implement, |
| namely LLVM, <a href="https://github.com/rust-lang/rustc_codegen_cranelift">Cranelift</a>, and <a href="https://github.com/rust-lang/rustc_codegen_gcc">GCC</a>.</p> |
| <p>Below is some background information on the refactoring that created this |
| abstract interface.</p> |
| <h2 id="refactoring-of-rustc_codegen_llvm"><a class="header" href="#refactoring-of-rustc_codegen_llvm">Refactoring of <code>rustc_codegen_llvm</code></a></h2> |
| <p>by Denis Merigoux, October 23rd 2018</p> |
| <h3 id="state-of-the-code-before-the-refactoring"><a class="header" href="#state-of-the-code-before-the-refactoring">State of the code before the refactoring</a></h3> |
| <p>All the code related to the compilation of MIR into LLVM IR was contained |
| inside the <code>rustc_codegen_llvm</code> crate. Here is the breakdown of the most |
| important elements:</p> |
| <ul> |
| <li>the <code>back</code> folder (7,800 LOC) implements the mechanisms for creating the |
| different object files and archive through LLVM, but also the communication |
| mechanisms for parallel code generation;</li> |
| <li>the <code>debuginfo</code> (3,200 LOC) folder contains all code that passes debug |
| information down to LLVM;</li> |
| <li>the <code>llvm</code> (2,200 LOC) folder defines the FFI necessary to communicate with |
| LLVM using the C++ API;</li> |
| <li>the <code>mir</code> (4,300 LOC) folder implements the actual lowering from MIR to LLVM |
| IR;</li> |
| <li>the <code>base.rs</code> (1,300 LOC) file contains some helper functions but also the |
| high-level code that launches the code generation and distributes the work.</li> |
| <li>the <code>builder.rs</code> (1,200 LOC) file contains all the functions generating |
| individual LLVM IR instructions inside a basic block;</li> |
| <li>the <code>common.rs</code> (450 LOC) contains various helper functions and all the |
| functions generating LLVM static values;</li> |
| <li>the <code>type_.rs</code> (300 LOC) defines most of the type translations to LLVM IR.</li> |
| </ul> |
| <p>The goal of this refactoring is to separate inside this crate code that is |
| specific to the LLVM from code that can be reused for other rustc backends. For |
| instance, the <code>mir</code> folder is almost entirely backend-specific but it relies |
| heavily on other parts of the crate. The separation of the code must not affect |
| the logic of the code nor its performance.</p> |
| <p>For these reasons, the separation process involves two transformations that |
| have to be done at the same time for the resulting code to compile:</p> |
| <ol> |
| <li>replace all the LLVM-specific types by generics inside function signatures |
| and structure definitions;</li> |
| <li>encapsulate all functions calling the LLVM FFI inside a set of traits that |
| will define the interface between backend-agnostic code and the backend.</li> |
| </ol> |
| <p>While the LLVM-specific code will be left in <code>rustc_codegen_llvm</code>, all the new |
| traits and backend-agnostic code will be moved in <code>rustc_codegen_ssa</code> (name |
| suggestion by @eddyb).</p> |
| <h3 id="generic-types-and-structures"><a class="header" href="#generic-types-and-structures">Generic types and structures</a></h3> |
| <p>@irinagpopa started to parametrize the types of <code>rustc_codegen_llvm</code> by a |
| generic <code>Value</code> type, implemented in LLVM by a reference <code>&'ll Value</code>. This |
| work has been extended to all structures inside the <code>mir</code> folder and elsewhere, |
| as well as for LLVM's <code>BasicBlock</code> and <code>Type</code> types.</p> |
| <p>The two most important structures for the LLVM codegen are <code>CodegenCx</code> and |
| <code>Builder</code>. They are parametrized by multiple lifetime parameters and the type |
| for <code>Value</code>.</p> |
| <pre><code class="language-rust ignore">struct CodegenCx<'ll, 'tcx> { |
| /* ... */ |
| } |
| |
| struct Builder<'a, 'll, 'tcx> { |
| cx: &'a CodegenCx<'ll, 'tcx>, |
| /* ... */ |
| }</code></pre> |
| <p><code>CodegenCx</code> is used to compile one codegen-unit that can contain multiple |
| functions, whereas <code>Builder</code> is created to compile one basic block.</p> |
| <p>The code in <code>rustc_codegen_llvm</code> has to deal with multiple explicit lifetime |
| parameters, that correspond to the following:</p> |
| <ul> |
| <li><code>'tcx</code> is the longest lifetime, that corresponds to the original <code>TyCtxt</code> |
| containing the program's information;</li> |
| <li><code>'a</code> is a short-lived reference of a <code>CodegenCx</code> or another object inside a |
| struct;</li> |
| <li><code>'ll</code> is the lifetime of references to LLVM objects such as <code>Value</code> or |
| <code>Type</code>.</li> |
| </ul> |
| <p>Although there are already many lifetime parameters in the code, making it |
| generic uncovered situations where the borrow-checker was passing only due to |
| the special nature of the LLVM objects manipulated (they are extern pointers). |
| For instance, an additional lifetime parameter had to be added to |
| <code>LocalAnalyser</code> in <code>analyse.rs</code>, leading to the definition:</p> |
| <pre><code class="language-rust ignore">struct LocalAnalyzer<'mir, 'a, 'tcx> { |
| /* ... */ |
| }</code></pre> |
| <p>However, the two most important structures <code>CodegenCx</code> and <code>Builder</code> are not |
| defined in the backend-agnostic code. Indeed, their content is highly specific |
| of the backend and it makes more sense to leave their definition to the backend |
| implementor than to allow just a narrow spot via a generic field for the |
| backend's context.</p> |
| <h3 id="traits-and-interface"><a class="header" href="#traits-and-interface">Traits and interface</a></h3> |
| <p>Because they have to be defined by the backend, <code>CodegenCx</code> and <code>Builder</code> will |
| be the structures implementing all the traits defining the backend's interface. |
| These traits are defined in the folder <code>rustc_codegen_ssa/traits</code> and all the |
| backend-agnostic code is parametrized by them. For instance, let us explain how |
| a function in <code>base.rs</code> is parametrized:</p> |
| <pre><code class="language-rust ignore">pub fn codegen_instance<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( |
| cx: &'a Bx::CodegenCx, |
| instance: Instance<'tcx> |
| ) { |
| /* ... */ |
| }</code></pre> |
| <p>In this signature, we have the two lifetime parameters explained earlier and |
| the master type <code>Bx</code> which satisfies the trait <code>BuilderMethods</code> corresponding |
| to the interface satisfied by the <code>Builder</code> struct. The <code>BuilderMethods</code> |
| defines an associated type <code>Bx::CodegenCx</code> that itself satisfies the |
| <code>CodegenMethods</code> traits implemented by the struct <code>CodegenCx</code>.</p> |
| <p>On the trait side, here is an example with part of the definition of |
| <code>BuilderMethods</code> in <code>traits/builder.rs</code>:</p> |
| <pre><code class="language-rust ignore">pub trait BuilderMethods<'a, 'tcx>: |
| HasCodegen<'tcx> |
| + DebugInfoBuilderMethods<'tcx> |
| + ArgTypeMethods<'tcx> |
| + AbiBuilderMethods<'tcx> |
| + IntrinsicCallMethods<'tcx> |
| + AsmBuilderMethods<'tcx> |
| { |
| fn new_block<'b>( |
| cx: &'a Self::CodegenCx, |
| llfn: Self::Function, |
| name: &'b str |
| ) -> Self; |
| /* ... */ |
| fn cond_br( |
| &mut self, |
| cond: Self::Value, |
| then_llbb: Self::BasicBlock, |
| else_llbb: Self::BasicBlock, |
| ); |
| /* ... */ |
| }</code></pre> |
| <p>Finally, a master structure implementing the <code>ExtraBackendMethods</code> trait is |
| used for high-level codegen-driving functions like <code>codegen_crate</code> in |
| <code>base.rs</code>. For LLVM, it is the empty <code>LlvmCodegenBackend</code>. |
| <code>ExtraBackendMethods</code> should be implemented by the same structure that |
| implements the <code>CodegenBackend</code> defined in |
| <code>rustc_codegen_utils/codegen_backend.rs</code>.</p> |
| <p>During the traitification process, certain functions have been converted from |
| methods of a local structure to methods of <code>CodegenCx</code> or <code>Builder</code> and a |
| corresponding <code>self</code> parameter has been added. Indeed, LLVM stores information |
| internally that it can access when called through its API. This information |
| does not show up in a Rust data structure carried around when these methods are |
| called. However, when implementing a Rust backend for <code>rustc</code>, these methods |
| will need information from <code>CodegenCx</code>, hence the additional parameter (unused |
| in the LLVM implementation of the trait).</p> |
| <h3 id="state-of-the-code-after-the-refactoring"><a class="header" href="#state-of-the-code-after-the-refactoring">State of the code after the refactoring</a></h3> |
| <p>The traits offer an API which is very similar to the API of LLVM. This is not |
| the best solution since LLVM has a very special way of doing things: when |
| adding another backend, the traits definition might be changed in order to |
| offer more flexibility.</p> |
| <p>However, the current separation between backend-agnostic and LLVM-specific code |
| has allowed the reuse of a significant part of the old <code>rustc_codegen_llvm</code>. |
| Here is the new LOC breakdown between backend-agnostic (BA) and LLVM for the |
| most important elements:</p> |
| <ul> |
| <li><code>back</code> folder: 3,800 (BA) vs 4,100 (LLVM);</li> |
| <li><code>mir</code> folder: 4,400 (BA) vs 0 (LLVM);</li> |
| <li><code>base.rs</code>: 1,100 (BA) vs 250 (LLVM);</li> |
| <li><code>builder.rs</code>: 1,400 (BA) vs 0 (LLVM);</li> |
| <li><code>common.rs</code>: 350 (BA) vs 350 (LLVM);</li> |
| </ul> |
| <p>The <code>debuginfo</code> folder has been left almost untouched by the splitting and is |
| specific to LLVM. Only its high-level features have been traitified.</p> |
| <p>The new <code>traits</code> folder has 1500 LOC only for trait definitions. Overall, the |
| 27,000 LOC-sized old <code>rustc_codegen_llvm</code> code has been split into the new |
| 18,500 LOC-sized new <code>rustc_codegen_llvm</code> and the 12,000 LOC-sized |
| <code>rustc_codegen_ssa</code>. We can say that this refactoring allowed the reuse of |
| approximately 10,000 LOC that would otherwise have had to be duplicated between |
| the multiple backends of <code>rustc</code>.</p> |
| <p>The refactored version of <code>rustc</code>'s backend introduced no regression over the |
| test suite nor in performance benchmark, which is in coherence with the nature |
| of the refactoring that used only compile-time parametricity (no trait |
| objects).</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="implicit-caller-location"><a class="header" href="#implicit-caller-location">Implicit caller location</a></h1> |
| <p>Approved in <a href="https://github.com/rust-lang/rfcs/blob/master/text/2091-inline-semantic.md">RFC 2091</a>, this feature enables the accurate reporting of caller location during panics |
| initiated from functions like <code>Option::unwrap</code>, <code>Result::expect</code>, and <code>Index::index</code>. This feature |
| adds the <a href="https://doc.rust-lang.org/reference/attributes/codegen.html#the-track_caller-attribute"><code>#[track_caller]</code></a> attribute for functions, the |
| <a href="https://doc.rust-lang.org/nightly/core/intrinsics/fn.caller_location.html"><code>caller_location</code></a> intrinsic, and the stabilization-friendly |
| <a href="https://doc.rust-lang.org/nightly/core/panic/struct.Location.html#method.caller"><code>core::panic::Location::caller</code></a> wrapper.</p> |
| <h2 id="motivating-example"><a class="header" href="#motivating-example">Motivating example</a></h2> |
| <p>Take this example program:</p> |
| <pre><pre class="playground"><code class="language-rust">fn main() { |
| let foo: Option<()> = None; |
| foo.unwrap(); // this should produce a useful panic message! |
| }</code></pre></pre> |
| <p>Prior to Rust 1.42, panics like this <code>unwrap()</code> printed a location in core:</p> |
| <pre><code>$ rustc +1.41.0 example.rs; example.exe |
| thread 'main' panicked at 'called `Option::unwrap()` on a `None` value',...core\macros\mod.rs:15:40 |
| note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace. |
| </code></pre> |
| <p>As of 1.42, we get a much more helpful message:</p> |
| <pre><code>$ rustc +1.42.0 example.rs; example.exe |
| thread 'main' panicked at 'called `Option::unwrap()` on a `None` value', example.rs:3:5 |
| note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace |
| </code></pre> |
| <p>These error messages are achieved through a combination of changes to <code>panic!</code> internals to make use |
| of <code>core::panic::Location::caller</code> and a number of <code>#[track_caller]</code> annotations in the standard |
| library which propagate caller information.</p> |
| <h2 id="reading-caller-location"><a class="header" href="#reading-caller-location">Reading caller location</a></h2> |
| <p>Previously, <code>panic!</code> made use of the <code>file!()</code>, <code>line!()</code>, and <code>column!()</code> macros to construct a |
| <a href="https://doc.rust-lang.org/core/panic/struct.Location.html"><code>Location</code></a> pointing to where the panic occurred. These macros couldn't be given an overridden |
| location, so functions which intentionally invoked <code>panic!</code> couldn't provide their own location, |
| hiding the actual source of error.</p> |
| <p>Internally, <code>panic!()</code> now calls <a href="https://doc.rust-lang.org/nightly/core/panic/struct.Location.html#method.caller"><code>core::panic::Location::caller()</code></a> to find out where it |
| was expanded. This function is itself annotated with <code>#[track_caller]</code> and wraps the |
| <a href="https://doc.rust-lang.org/nightly/core/intrinsics/fn.caller_location.html"><code>caller_location</code></a> compiler intrinsic implemented by rustc. This intrinsic is easiest |
| explained in terms of how it works in a <code>const</code> context.</p> |
| <h2 id="caller-location-in-const"><a class="header" href="#caller-location-in-const">Caller location in <code>const</code></a></h2> |
| <p>There are two main phases to returning the caller location in a const context: walking up the stack |
| to find the right location and allocating a const value to return.</p> |
| <h3 id="finding-the-right-location"><a class="header" href="#finding-the-right-location">Finding the right <code>Location</code></a></h3> |
| <p>In a const context we "walk up the stack" from where the intrinsic is invoked, stopping when we |
| reach the first function call in the stack which does <em>not</em> have the attribute. This walk is in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.InterpCx.html#method.find_closest_untracked_caller_location"><code>InterpCx::find_closest_untracked_caller_location()</code></a>.</p> |
| <p>Starting at the bottom, we iterate up over stack <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.Frame.html"><code>Frame</code></a>s in the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.InterpCx.html#structfield.stack"><code>InterpCx::stack</code></a>, calling |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/instance/enum.InstanceKind.html#method.requires_caller_location"><code>InstanceKind::requires_caller_location</code></a> on the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.Frame.html#structfield.instance"><code>Instance</code>s from each <code>Frame</code></a>. We stop once we find one that returns <code>false</code> and |
| return the span of the <em>previous</em> frame which was the "topmost" tracked function.</p> |
| <h3 id="allocating-a-static-location"><a class="header" href="#allocating-a-static-location">Allocating a static <code>Location</code></a></h3> |
| <p>Once we have a <code>Span</code>, we need to allocate static memory for the <code>Location</code>, which is performed by |
| the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.const_caller_location"><code>TyCtxt::const_caller_location()</code></a> query. Internally this calls |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.InterpCx.html#method.alloc_caller_location"><code>InterpCx::alloc_caller_location()</code></a> and results in a unique |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/enum.MemoryKind.html#variant.CallerLocation">memory kind</a> (<code>MemoryKind::CallerLocation</code>). The SSA codegen backend is able |
| to emit code for these same values, and we use this code there as well.</p> |
| <p>Once our <code>Location</code> has been allocated in static memory, our intrinsic returns a reference to it.</p> |
| <h2 id="generating-code-for-track_caller-callees"><a class="header" href="#generating-code-for-track_caller-callees">Generating code for <code>#[track_caller]</code> callees</a></h2> |
| <p>To generate efficient code for a tracked function and its callers, we need to provide the same |
| behavior from the intrinsic's point of view without having a stack to walk up at runtime. We invert |
| the approach: as we grow the stack down we pass an additional argument to calls of tracked functions |
| rather than walking up the stack when the intrinsic is called. That additional argument can be |
| returned wherever the caller location is queried.</p> |
| <p>The argument we append is of type <code>&'static core::panic::Location<'static></code>. A reference was chosen |
| to avoid unnecessary copying because a pointer is a third the size of |
| <code>std::mem::size_of::<core::panic::Location>() == 24</code> at time of writing.</p> |
| <p>When generating a call to a function which is tracked, we pass the location argument the value of |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#method.get_caller_location"><code>FunctionCx::get_caller_location</code></a>.</p> |
| <p>If the calling function is tracked, <code>get_caller_location</code> returns the local in |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#structfield.caller_location"><code>FunctionCx::caller_location</code></a> which was populated by the current caller's caller. |
| In these cases the intrinsic "returns" a reference which was actually provided in an argument to its |
| caller.</p> |
| <p>If the calling function is not tracked, <code>get_caller_location</code> allocates a <code>Location</code> static from |
| the current <code>Span</code> and returns a reference to that.</p> |
| <p>We more efficiently achieve the same behavior as a loop starting from the bottom by passing a single |
| <code>&Location</code> value through the <code>caller_location</code> fields of multiple <code>FunctionCx</code>s as we grow the |
| stack downward.</p> |
| <h3 id="codegen-examples"><a class="header" href="#codegen-examples">Codegen examples</a></h3> |
| <p>What does this transformation look like in practice? Take this example which uses the new feature:</p> |
| <pre><pre class="playground"><code class="language-rust">#![feature(track_caller)] |
| use std::panic::Location; |
| |
| #[track_caller] |
| fn print_caller() { |
| println!("called from {}", Location::caller()); |
| } |
| |
| fn main() { |
| print_caller(); |
| }</code></pre></pre> |
| <p>Here <code>print_caller()</code> appears to take no arguments, but we compile it to something like this:</p> |
| <pre><pre class="playground"><code class="language-rust">#![feature(panic_internals)] |
| use std::panic::Location; |
| |
| fn print_caller(caller: &Location) { |
| println!("called from {}", caller); |
| } |
| |
| fn main() { |
| print_caller(&Location::internal_constructor(file!(), line!(), column!())); |
| }</code></pre></pre> |
| <h3 id="dynamic-dispatch"><a class="header" href="#dynamic-dispatch">Dynamic dispatch</a></h3> |
| <p>In codegen contexts we have to modify the callee ABI to pass this information down the stack, but |
| the attribute expressly does <em>not</em> modify the type of the function. The ABI change must be |
| transparent to type checking and remain sound in all uses.</p> |
| <p>Direct calls to tracked functions will always know the full codegen flags for the callee and can |
| generate appropriate code. Indirect callers won't have this information and it's not encoded in |
| the type of the function pointer they call, so we generate a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.InstanceKind.html#variant.ReifyShim"><code>ReifyShim</code></a> around the function |
| whenever taking a pointer to it. This shim isn't able to report the actual location of the indirect |
| call (the function's definition site is reported instead), but it prevents miscompilation and is |
| probably the best we can do without modifying fully-stabilized type signatures.</p> |
| <blockquote> |
| <p><em>Note:</em> We always emit a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.InstanceKind.html#variant.ReifyShim"><code>ReifyShim</code></a> when taking a pointer to a tracked function. While the |
| constraint here is imposed by codegen contexts, we don't know during MIR construction of the shim |
| whether we'll be called in a const context (safe to ignore shim) or in a codegen context (unsafe |
| to ignore shim). Even if we did know, the results from const and codegen contexts must agree.</p> |
| </blockquote> |
| <h2 id="the-attribute"><a class="header" href="#the-attribute">The attribute</a></h2> |
| <p>The <code>#[track_caller]</code> attribute is checked alongside other codegen attributes to ensure the |
| function:</p> |
| <ul> |
| <li>has the <code>"Rust"</code> ABI (as opposed to e.g., <code>"C"</code>)</li> |
| <li>is not a closure</li> |
| <li>is not <code>#[naked]</code></li> |
| </ul> |
| <p>If the use is valid, we set <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/middle/codegen_fn_attrs/struct.CodegenFnAttrFlags.html#associatedconstant.TRACK_CALLER"><code>CodegenFnAttrsFlags::TRACK_CALLER</code></a>. This flag influences |
| the return value of <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/instance/enum.InstanceKind.html#method.requires_caller_location"><code>InstanceKind::requires_caller_location</code></a> which is in turn |
| used in both const and codegen contexts to ensure correct propagation.</p> |
| <h3 id="traits"><a class="header" href="#traits">Traits</a></h3> |
| <p>When applied to trait method implementations, the attribute works as it does for regular functions.</p> |
| <p>When applied to a trait method prototype, the attribute applies to all implementations of the |
| method. When applied to a default trait method implementation, the attribute takes effect on |
| that implementation <em>and</em> any overrides.</p> |
| <p>Examples:</p> |
| <pre><pre class="playground"><code class="language-rust">#![feature(track_caller)] |
| |
| macro_rules! assert_tracked { |
| () => {{ |
| let location = std::panic::Location::caller(); |
| assert_eq!(location.file(), file!()); |
| assert_ne!(location.line(), line!(), "line should be outside this fn"); |
| println!("called at {}", location); |
| }}; |
| } |
| |
| trait TrackedFourWays { |
| /// All implementations inherit `#[track_caller]`. |
| #[track_caller] |
| fn blanket_tracked(); |
| |
| /// Implementors can annotate themselves. |
| fn local_tracked(); |
| |
| /// This implementation is tracked (overrides are too). |
| #[track_caller] |
| fn default_tracked() { |
| assert_tracked!(); |
| } |
| |
| /// Overrides of this implementation are tracked (it is too). |
| #[track_caller] |
| fn default_tracked_to_override() { |
| assert_tracked!(); |
| } |
| } |
| |
| /// This impl uses the default impl for `default_tracked` and provides its own for |
| /// `default_tracked_to_override`. |
| impl TrackedFourWays for () { |
| fn blanket_tracked() { |
| assert_tracked!(); |
| } |
| |
| #[track_caller] |
| fn local_tracked() { |
| assert_tracked!(); |
| } |
| |
| fn default_tracked_to_override() { |
| assert_tracked!(); |
| } |
| } |
| |
| fn main() { |
| <() as TrackedFourWays>::blanket_tracked(); |
| <() as TrackedFourWays>::default_tracked(); |
| <() as TrackedFourWays>::default_tracked_to_override(); |
| <() as TrackedFourWays>::local_tracked(); |
| }</code></pre></pre> |
| <h2 id="backgroundhistory"><a class="header" href="#backgroundhistory">Background/History</a></h2> |
| <p>Broadly speaking, this feature's goal is to improve common Rust error messages without breaking |
| stability guarantees, requiring modifications to end-user source, relying on platform-specific |
| debug-info, or preventing user-defined types from having the same error-reporting benefits.</p> |
| <p>Improving the output of these panics has been a goal of proposals since at least mid-2016 (see |
| <a href="https://github.com/rust-lang/rfcs/blob/master/text/2091-inline-semantic.md#non-viable-alternatives">non-viable alternatives</a> in the approved RFC for details). It took two more years until RFC 2091 |
| was approved, much of its <a href="https://github.com/rust-lang/rfcs/blob/master/text/2091-inline-semantic.md#rationale">rationale</a> for this feature's design having been discovered through the |
| discussion around several earlier proposals.</p> |
| <p>The design in the original RFC limited itself to implementations that could be done inside the |
| compiler at the time without significant refactoring. However in the year and a half between the |
| approval of the RFC and the actual implementation work, a <a href="https://github.com/rust-lang/rust/issues/47809#issuecomment-443538059">revised design</a> was proposed and written |
| up on the tracking issue. During the course of implementing that, it was also discovered that an |
| implementation was possible without modifying the number of arguments in a function's MIR, which |
| would simplify later stages and unlock use in traits.</p> |
| <p>Because the RFC's implementation strategy could not readily support traits, the semantics were not |
| originally specified. They have since been implemented following the path which seemed most correct |
| to the author and reviewers.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="libraries-and-metadata"><a class="header" href="#libraries-and-metadata">Libraries and metadata</a></h1> |
| <p>When the compiler sees a reference to an external crate, it needs to load some |
| information about that crate. This chapter gives an overview of that process, |
| and the supported file formats for crate libraries.</p> |
| <h2 id="libraries"><a class="header" href="#libraries">Libraries</a></h2> |
| <p>A crate dependency can be loaded from an <code>rlib</code>, <code>dylib</code>, or <code>rmeta</code> file. A |
| key point of these file formats is that they contain <code>rustc</code>-specific |
| <a href="backend/libs-and-metadata.html#metadata"><em>metadata</em></a>. This metadata allows the compiler to discover enough |
| information about the external crate to understand the items it contains, |
| which macros it exports, and <em>much</em> more.</p> |
| <h3 id="rlib"><a class="header" href="#rlib">rlib</a></h3> |
| <p>An <code>rlib</code> is an <a href="https://en.wikipedia.org/wiki/Ar_(Unix)">archive file</a>, which is similar to a tar file. This file |
| format is specific to <code>rustc</code>, and may change over time. This file contains:</p> |
| <ul> |
| <li>Object code, which is the result of code generation. This is used during |
| regular linking. There is a separate <code>.o</code> file for each <a href="backend/../backend/codegen.html">codegen unit</a>. The |
| codegen step can be skipped with the <a href="https://doc.rust-lang.org/rustc/codegen-options/index.html#linker-plugin-lto"><code>-C linker-plugin-lto</code></a> CLI option, which means each <code>.o</code> |
| file will only contain LLVM bitcode.</li> |
| <li><a href="https://llvm.org/docs/BitCodeFormat.html">LLVM bitcode</a>, which is a binary representation of LLVM's intermediate |
| representation, which is embedded as a section in the <code>.o</code> files. This can |
| be used for <a href="https://llvm.org/docs/LinkTimeOptimization.html">Link Time Optimization</a> (LTO). This can be removed with the |
| <a href="https://doc.rust-lang.org/rustc/codegen-options/index.html#embed-bitcode"><code>-C embed-bitcode=no</code></a> CLI option to improve compile times |
| and reduce disk space if LTO is not needed.</li> |
| <li><code>rustc</code> <a href="backend/libs-and-metadata.html#metadata">metadata</a>, in a file named <code>lib.rmeta</code>.</li> |
| <li>A symbol table, which is essentially a list of symbols with offsets to the |
| object files that contain that symbol. This is pretty standard for archive |
| files.</li> |
| </ul> |
| <h3 id="dylib"><a class="header" href="#dylib">dylib</a></h3> |
| <p>A <code>dylib</code> is a platform-specific shared library. It includes the <code>rustc</code> |
| <a href="backend/libs-and-metadata.html#metadata">metadata</a> in a special link section called <code>.rustc</code>.</p> |
| <h3 id="rmeta"><a class="header" href="#rmeta">rmeta</a></h3> |
| <p>An <code>rmeta</code> file is a custom binary format that contains the <a href="backend/libs-and-metadata.html#metadata">metadata</a> for the |
| crate. This file can be used for fast "checks" of a project by skipping all code |
| generation (as is done with <code>cargo check</code>), collecting enough information for |
| documentation (as is done with <code>cargo doc</code>), or for <a href="backend/libs-and-metadata.html#pipelining">pipelining</a>. |
| This file is created if the <a href="https://doc.rust-lang.org/rustc/command-line-arguments.html#option-emit"><code>--emit=metadata</code></a> CLI option is used.</p> |
| <p><code>rmeta</code> files do not support linking, since they do not contain compiled |
| object files.</p> |
| <h2 id="metadata"><a class="header" href="#metadata">Metadata</a></h2> |
| <p>The metadata contains a wide swath of different elements. This guide will not go |
| into detail about every field it contains. You are encouraged to browse the |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/struct.CrateRoot.html"><code>CrateRoot</code></a> definition to get a sense of the different elements it contains. |
| Everything about metadata encoding and decoding is in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/index.html"><code>rustc_metadata</code></a> |
| package.</p> |
| <p>Here are a few highlights of things it contains:</p> |
| <ul> |
| <li>The version of the <code>rustc</code> compiler. The compiler will refuse to load files |
| from any other version.</li> |
| <li>The <a href="backend/libs-and-metadata.html#strict-version-hash">Strict Version Hash</a> (SVH). This helps ensure the |
| correct dependency is loaded.</li> |
| <li>The <a href="backend/libs-and-metadata.html#stable-crate-id">Stable Crate Id</a>. This is a hash used |
| to identify crates.</li> |
| <li>Information about all the source files in the library. This can be used for |
| a variety of things, such as diagnostics pointing to sources in a |
| dependency.</li> |
| <li>Information about exported macros, traits, types, and items. Generally, |
| anything that's needed to be known when a path references something inside a |
| crate dependency.</li> |
| <li>Encoded <a href="backend/../mir/index.html">MIR</a>. This is optional, and only encoded if needed for code |
| generation. <code>cargo check</code> skips this for performance reasons.</li> |
| </ul> |
| <h3 id="strict-version-hash"><a class="header" href="#strict-version-hash">Strict Version Hash</a></h3> |
| <p>The Strict Version Hash (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/svh/struct.Svh.html">SVH</a>, also known as the "crate hash") is a 64-bit |
| hash that is used to ensure that the correct crate dependencies are loaded. It |
| is possible for a directory to contain multiple copies of the same dependency |
| built with different settings, or built from different sources. The crate |
| loader will skip any crates that have the wrong SVH.</p> |
| <p>The SVH is also used for the <a href="backend/../queries/incremental-compilation.html">incremental compilation</a> session filename, |
| though that usage is mostly historic.</p> |
| <p>The hash includes a variety of elements:</p> |
| <ul> |
| <li>Hashes of the HIR nodes.</li> |
| <li>All of the upstream crate hashes.</li> |
| <li>All of the source filenames.</li> |
| <li>Hashes of certain command-line flags (like <code>-C metadata</code> via the <a href="backend/libs-and-metadata.html#stable-crate-id">Stable |
| Crate Id</a>, and all CLI options marked with <code>[TRACKED]</code>).</li> |
| </ul> |
| <p>See <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast_lowering/fn.compute_hir_hash.html"><code>compute_hir_hash</code></a> for where the hash is actually computed.</p> |
| <h3 id="stable-crate-id"><a class="header" href="#stable-crate-id">Stable Crate Id</a></h3> |
| <p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/def_id/struct.StableCrateId.html"><code>StableCrateId</code></a> is a 64-bit hash used to identify different crates with |
| potentially the same name. It is a hash of the crate name and all the |
| <a href="https://doc.rust-lang.org/rustc/codegen-options/index.html#metadata"><code>-C metadata</code></a> CLI options computed in <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/def_id/struct.StableCrateId.html#method.new"><code>StableCrateId::new</code></a>. It is |
| used in a variety of places, such as symbol name mangling, crate loading, and |
| much more.</p> |
| <p>By default, all Rust symbols are mangled and incorporate the stable crate id. |
| This allows multiple versions of the same crate to be included together. Cargo |
| automatically generates <code>-C metadata</code> hashes based on a variety of factors, like |
| the package version, source, and target kind (a lib and test can have the same |
| crate name, so they need to be disambiguated).</p> |
| <h2 id="crate-loading"><a class="header" href="#crate-loading">Crate loading</a></h2> |
| <p>Crate loading can have quite a few subtle complexities. During <a href="backend/../name-resolution.html">name |
| resolution</a>, when an external crate is referenced (via an <code>extern crate</code> or |
| path), the resolver uses the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/creader/struct.CrateLoader.html"><code>CrateLoader</code></a> which is responsible for finding |
| the crate libraries and loading the <a href="backend/libs-and-metadata.html#metadata">metadata</a> for them. After the dependency |
| is loaded, the <code>CrateLoader</code> will provide the information the resolver needs |
| to perform its job (such as expanding macros, resolving paths, etc.).</p> |
| <p>To load each external crate, the <code>CrateLoader</code> uses a <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/locator/struct.CrateLocator.html"><code>CrateLocator</code></a> to |
| actually find the correct files for one specific crate. There is some great |
| documentation in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/locator/index.html"><code>locator</code></a> module that goes into detail on how loading |
| works, and I strongly suggest reading it to get the full picture.</p> |
| <p>The location of a dependency can come from several different places. Direct |
| dependencies are usually passed with <code>--extern</code> flags, and the loader can look |
| at those directly. Direct dependencies often have references to their own |
| dependencies, which need to be loaded, too. These are usually found by |
| scanning the directories passed with the <code>-L</code> flag for any file whose metadata |
| contains a matching crate name and <a href="backend/libs-and-metadata.html#strict-version-hash">SVH</a>. The loader |
| will also look at the <a href="backend/../building/bootstrapping/what-bootstrapping-does.html#what-is-a-sysroot">sysroot</a> to find dependencies.</p> |
| <p>As crates are loaded, they are kept in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/creader/struct.CStore.html"><code>CStore</code></a> with the crate metadata |
| wrapped in the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/decoder/struct.CrateMetadata.html"><code>CrateMetadata</code></a> struct. After resolution and expansion, the |
| <code>CStore</code> will make its way into the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.GlobalCtxt.html"><code>GlobalCtxt</code></a> for the rest of the |
| compilation.</p> |
| <h2 id="pipelining"><a class="header" href="#pipelining">Pipelining</a></h2> |
| <p>One trick to improve compile times is to start building a crate as soon as the |
| metadata for its dependencies is available. For a library, there is no need to |
| wait for the code generation of dependencies to finish. Cargo implements this |
| technique by telling <code>rustc</code> to emit an <a href="backend/libs-and-metadata.html#rmeta"><code>rmeta</code></a> file for each |
| dependency as well as an <a href="backend/libs-and-metadata.html#rlib"><code>rlib</code></a>. As early as it can, <code>rustc</code> will |
| save the <code>rmeta</code> file to disk before it continues to the code generation |
| phase. The compiler sends a JSON message to let the build tool know that it |
| can start building the next crate if possible.</p> |
| <p>The <a href="backend/libs-and-metadata.html#crate-loading">crate loading</a> system is smart enough to know when it |
| sees an <code>rmeta</code> file to use that if the <code>rlib</code> is not there (or has only been |
| partially written).</p> |
| <p>This pipelining isn't possible for binaries, because the linking phase will |
| require the code generation of all its dependencies. In the future, it may be |
| possible to further improve this scenario by splitting linking into a separate |
| command (see <a href="https://github.com/rust-lang/rust/issues/64191">#64191</a>).</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="profile-guided-optimization-1"><a class="header" href="#profile-guided-optimization-1">Profile-guided optimization</a></h1> |
| <p><code>rustc</code> supports doing profile-guided optimization (PGO). |
| This chapter describes what PGO is and how the support for it is |
| implemented in <code>rustc</code>.</p> |
| <h2 id="what-is-profiled-guided-optimization"><a class="header" href="#what-is-profiled-guided-optimization">What is profiled-guided optimization?</a></h2> |
| <p>The basic concept of PGO is to collect data about the typical execution of |
| a program (e.g. which branches it is likely to take) and then use this data |
| to inform optimizations such as inlining, machine-code layout, |
| register allocation, etc.</p> |
| <p>There are different ways of collecting data about a program's execution. |
| One is to run the program inside a profiler (such as <code>perf</code>) and another |
| is to create an instrumented binary, that is, a binary that has data |
| collection built into it, and run that. |
| The latter usually provides more accurate data.</p> |
| <h2 id="how-is-pgo-implemented-in-rustc"><a class="header" href="#how-is-pgo-implemented-in-rustc">How is PGO implemented in <code>rustc</code>?</a></h2> |
| <p><code>rustc</code> current PGO implementation relies entirely on LLVM. |
| LLVM actually <a href="https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization">supports multiple forms</a> of PGO:</p> |
| <ul> |
| <li>Sampling-based PGO where an external profiling tool like <code>perf</code> is used |
| to collect data about a program's execution.</li> |
| <li>GCOV-based profiling, where code coverage infrastructure is used to collect |
| profiling information.</li> |
| <li>Front-end based instrumentation, where the compiler front-end (e.g. Clang) |
| inserts instrumentation intrinsics into the LLVM IR it generates (but see the |
| <sup class="footnote-reference" id="fr-note-instrument-coverage-1"><a href="#footnote-note-instrument-coverage">1</a></sup>"Note").</li> |
| <li>IR-level instrumentation, where LLVM inserts the instrumentation intrinsics |
| itself during optimization passes.</li> |
| </ul> |
| <p><code>rustc</code> supports only the last approach, IR-level instrumentation, mainly |
| because it is almost exclusively implemented in LLVM and needs little |
| maintenance on the Rust side. Fortunately, it is also the most modern approach, |
| yielding the best results.</p> |
| <p>So, we are dealing with an instrumentation-based approach, i.e. profiling data |
| is generated by a specially instrumented version of the program that's being |
| optimized. Instrumentation-based PGO has two components: a compile-time |
| component and run-time component, and one needs to understand the overall |
| workflow to see how they interact.</p> |
| <h3 id="overall-workflow"><a class="header" href="#overall-workflow">Overall workflow</a></h3> |
| <p>Generating a PGO-optimized program involves the following four steps:</p> |
| <ol> |
| <li>Compile the program with instrumentation enabled (e.g. <code>rustc -C profile-generate main.rs</code>)</li> |
| <li>Run the instrumented program (e.g. <code>./main</code>) which generates a <code>default-<id>.profraw</code> file</li> |
| <li>Convert the <code>.profraw</code> file into a <code>.profdata</code> file using LLVM's <code>llvm-profdata</code> tool.</li> |
| <li>Compile the program again, this time making use of the profiling data |
| (e.g. <code>rustc -C profile-use=merged.profdata main.rs</code>)</li> |
| </ol> |
| <h3 id="compile-time-aspects"><a class="header" href="#compile-time-aspects">Compile-time aspects</a></h3> |
| <p>Depending on which step in the above workflow we are in, two different things |
| can happen at compile time:</p> |
| <h4 id="create-binaries-with-instrumentation"><a class="header" href="#create-binaries-with-instrumentation">Create binaries with instrumentation</a></h4> |
| <p>As mentioned above, the profiling instrumentation is added by LLVM. |
| <code>rustc</code> instructs LLVM to do so <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/rustllvm/PassWrapper.cpp#L412-L416">by setting the appropriate</a> |
| flags when creating LLVM <code>PassManager</code>s:</p> |
| <pre><code class="language-C"> // `PMBR` is an `LLVMPassManagerBuilderRef` |
| unwrap(PMBR)->EnablePGOInstrGen = true; |
| // Instrumented binaries have a default output path for the `.profraw` file |
| // hard-coded into them: |
| unwrap(PMBR)->PGOInstrGen = PGOGenPath; |
| </code></pre> |
| <p><code>rustc</code> also has to make sure that some of the symbols from LLVM's profiling |
| runtime are not removed <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/librustc_codegen_ssa/back/symbol_export.rs#L212-L225">by marking the with the right export level</a>.</p> |
| <h4 id="compile-binaries-where-optimizations-make-use-of-profiling-data"><a class="header" href="#compile-binaries-where-optimizations-make-use-of-profiling-data">Compile binaries where optimizations make use of profiling data</a></h4> |
| <p>In the final step of the workflow described above, the program is compiled |
| again, with the compiler using the gathered profiling data in order to drive |
| optimization decisions. <code>rustc</code> again leaves most of the work to LLVM here, |
| basically <a href="https://github.com/rust-lang/rust/blob/1.34.1/src/rustllvm/PassWrapper.cpp#L417-L420">just telling</a> the LLVM <code>PassManagerBuilder</code> |
| where the profiling data can be found:</p> |
| <pre><code class="language-C"> unwrap(PMBR)->PGOInstrUse = PGOUsePath; |
| </code></pre> |
| <p>LLVM does the rest (e.g. setting branch weights, marking functions with |
| <code>cold</code> or <code>inlinehint</code>, etc).</p> |
| <h3 id="runtime-aspects"><a class="header" href="#runtime-aspects">Runtime aspects</a></h3> |
| <p>Instrumentation-based approaches always also have a runtime component, i.e. |
| once we have an instrumented program, that program needs to be run in order |
| to generate profiling data, and collecting and persisting this profiling |
| data needs some infrastructure in place.</p> |
| <p>In the case of LLVM, these runtime components are implemented in |
| <a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/profile">compiler-rt</a> and statically linked into any instrumented |
| binaries. |
| The <code>rustc</code> version of this can be found in <code>library/profiler_builtins</code> which |
| basically packs the C code from <code>compiler-rt</code> into a Rust crate.</p> |
| <p>In order for <code>profiler_builtins</code> to be built, <code>profiler = true</code> must be set |
| in <code>rustc</code>'s <code>bootstrap.toml</code>.</p> |
| <h2 id="testing-pgo"><a class="header" href="#testing-pgo">Testing PGO</a></h2> |
| <p>Since the PGO workflow spans multiple compiler invocations most testing happens |
| in <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/run-make">run-make tests</a> (the relevant tests have <code>pgo</code> in their name). |
| There is also a <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/codegen-llvm/pgo-instrumentation.rs">codegen test</a> that checks that some expected |
| instrumentation artifacts show up in LLVM IR.</p> |
| <h2 id="additional-information"><a class="header" href="#additional-information">Additional information</a></h2> |
| <p>Clang's documentation contains a good overview on <a href="https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization">PGO in LLVM</a>.</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-note-instrument-coverage"> |
| <p>Note: <code>rustc</code> now supports front-end-based coverage |
| instrumentation, via the experimental option |
| <a href="./llvm-coverage-instrumentation.html"><code>-C instrument-coverage</code></a>, but using these |
| coverage results for PGO has not been attempted at this time. <a href="#fr-note-instrument-coverage-1">↩</a></p> |
| </li> |
| </ol><div style="break-before: page; page-break-before: always;"></div><h1 id="llvm-source-based-code-coverage"><a class="header" href="#llvm-source-based-code-coverage">LLVM source-based code coverage</a></h1> |
| <p><code>rustc</code> supports detailed source-based code and test coverage analysis |
| with a command line option (<code>-C instrument-coverage</code>) that instruments Rust |
| libraries and binaries with additional instructions and data, at compile time.</p> |
| <p>The coverage instrumentation injects calls to the LLVM intrinsic instruction |
| <a href="https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic"><code>llvm.instrprof.increment</code></a> at code branches |
| (based on a MIR-based control flow analysis), and LLVM converts these to |
| instructions that increment static counters, when executed. The LLVM coverage |
| instrumentation also requires a <a href="https://llvm.org/docs/CoverageMappingFormat.html">Coverage Map</a> that encodes source metadata, |
| mapping counter IDs--directly and indirectly--to the file locations (with |
| start and end line and column).</p> |
| <p>Rust libraries, with or without coverage instrumentation, can be linked into |
| instrumented binaries. When the program is executed and cleanly terminates, |
| LLVM libraries write the final counter values to a file (<code>default.profraw</code> or |
| a custom file set through environment variable <code>LLVM_PROFILE_FILE</code>).</p> |
| <p>Developers use existing LLVM coverage analysis tools to decode <code>.profraw</code> |
| files, with corresponding Coverage Maps (from matching binaries that produced |
| them), and generate various reports for analysis, for example:</p> |
| <p><img alt="Screenshot of sample `llvm-cov show` result, for function add_quoted_string" |
| src="img/llvm-cov-show-01.png" class="center"/> |
| <br/></p> |
| <p>Detailed instructions and examples are documented in the |
| <a href="https://doc.rust-lang.org/nightly/rustc/instrument-coverage.html">rustc book</a>.</p> |
| <h2 id="recommended-bootstraptoml-settings"><a class="header" href="#recommended-bootstraptoml-settings">Recommended <code>bootstrap.toml</code> settings</a></h2> |
| <p>When working on the coverage instrumentation code, it is usually necessary to |
| <strong>enable the profiler runtime</strong> by setting <code>profiler = true</code> in <code>[build]</code>. |
| This allows the compiler to produce instrumented binaries, and makes it possible |
| to run the full coverage test suite.</p> |
| <p>Enabling debug assertions in the compiler and in LLVM is recommended, but not |
| mandatory.</p> |
| <pre><code class="language-toml"># Similar to the "compiler" profile, but also enables debug assertions in LLVM. |
| # These assertions can detect malformed coverage mappings in some cases. |
| profile = "codegen" |
| |
| [build] |
| # IMPORTANT: This tells the build system to build the LLVM profiler runtime. |
| # Without it, the compiler can't produce coverage-instrumented binaries, |
| # and many of the coverage tests will be skipped. |
| profiler = true |
| |
| [rust] |
| # Enable debug assertions in the compiler. |
| debug-assertions = true |
| </code></pre> |
| <h2 id="rust-symbol-mangling"><a class="header" href="#rust-symbol-mangling">Rust symbol mangling</a></h2> |
| <p><code>-C instrument-coverage</code> automatically enables Rust symbol mangling <code>v0</code> (as |
| if the user specified <code>-C symbol-mangling-version=v0</code> option when invoking |
| <code>rustc</code>) to ensure consistent and reversible name mangling. This has two |
| important benefits:</p> |
| <ol> |
| <li>LLVM coverage tools can analyze coverage over multiple runs, including some |
| changes to source code; so mangled names must be consistent across compilations.</li> |
| <li>LLVM coverage reports can report coverage by function, and even separates |
| out the coverage counts of each unique instantiation of a generic function, |
| if invoked with multiple type substitution variations.</li> |
| </ol> |
| <h2 id="the-llvm-profiler-runtime"><a class="header" href="#the-llvm-profiler-runtime">The LLVM profiler runtime</a></h2> |
| <p>Coverage data is only generated by running the executable Rust program. <code>rustc</code> |
| statically links coverage-instrumented binaries with LLVM runtime code |
| (<a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/profile">compiler-rt</a>) that implements program hooks |
| (such as an <code>exit</code> hook) to write the counter values to the <code>.profraw</code> file.</p> |
| <p>In the <code>rustc</code> source tree, |
| <code>library/profiler_builtins</code> bundles the LLVM <code>compiler-rt</code> code into a Rust library crate. |
| Note that when building <code>rustc</code>, |
| <code>profiler_builtins</code> is only included when <code>build.profiler = true</code> is set in <code>bootstrap.toml</code>.</p> |
| <p>When compiling with <code>-C instrument-coverage</code>, |
| <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/creader/struct.CrateLoader.html#method.postprocess"><code>CrateLoader::postprocess()</code></a> dynamically loads |
| <code>profiler_builtins</code> by calling <code>inject_profiler_runtime()</code>.</p> |
| <h2 id="testing-coverage-instrumentation"><a class="header" href="#testing-coverage-instrumentation">Testing coverage instrumentation</a></h2> |
| <p><a href="./tests/compiletest.html#coverage-tests">(See also the compiletest documentation for the <code>tests/coverage</code> |
| test suite.)</a></p> |
| <p>Coverage instrumentation in the MIR is validated by a <code>mir-opt</code> test: |
| <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/mir-opt/coverage/instrument_coverage.rs"><code>tests/mir-opt/coverage/instrument_coverage.rs</code></a>.</p> |
| <p>Coverage instrumentation in LLVM IR is validated by the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/coverage"><code>tests/coverage</code></a> |
| test suite in <code>coverage-map</code> mode. |
| These tests compile a test program to LLVM IR assembly, and then |
| use the <a href="https://github.com/rust-lang/rust/tree/HEAD/src/tools/coverage-dump"><code>src/tools/coverage-dump</code></a> tool to extract and pretty-print the |
| coverage mappings that would be embedded in the final binary.</p> |
| <p>End-to-end testing of coverage instrumentation and coverage reporting is |
| performed by the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/coverage"><code>tests/coverage</code></a> test suite in <code>coverage-run</code> mode, |
| and by the <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/coverage-run-rustdoc"><code>tests/coverage-run-rustdoc</code></a> test suite. |
| These tests compile and run a test program with coverage |
| instrumentation, then use LLVM tools to convert the coverage data into a |
| human-readable coverage report.</p> |
| <blockquote> |
| <p>Tests in <code>coverage-run</code> mode have an implicit <code>//@ needs-profiler-runtime</code> |
| directive, so they will be skipped if the profiler runtime has not been |
| <a href="llvm-coverage-instrumentation.html#recommended-configtoml-settings">enabled in <code>bootstrap.toml</code></a>.</p> |
| </blockquote> |
| <p>Finally, the <a href="https://github.com/rust-lang/rust/blob/HEAD/tests/mir-opt/coverage/instrument_coverage.rs"><code>tests/codegen-llvm/instrument-coverage/testprog.rs</code></a> test compiles a simple Rust program |
| with <code>-C instrument-coverage</code> and compares the compiled program's LLVM IR to |
| expected LLVM IR instructions and structured data for a coverage-enabled |
| program, including various checks for Coverage Map-related metadata and the LLVM |
| intrinsic calls to increment the runtime counters.</p> |
| <p>Expected results for the <code>coverage</code>, <code>coverage-run-rustdoc</code>, |
| and <code>mir-opt</code> tests can be refreshed by running:</p> |
| <pre><code class="language-shell">./x test coverage --bless |
| ./x test coverage-run-rustdoc --bless |
| ./x test tests/mir-opt --bless |
| </code></pre> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="sanitizers-support"><a class="header" href="#sanitizers-support">Sanitizers support</a></h1> |
| <p>The rustc compiler contains support for following sanitizers:</p> |
| <ul> |
| <li><a href="https://clang.llvm.org/docs/AddressSanitizer.html">AddressSanitizer</a> a faster memory error detector. |
| Can detect out-of-bounds access to heap, stack, and globals, use after free, use |
| after return, double free, invalid free, memory leaks.</li> |
| <li><a href="https://clang.llvm.org/docs/ControlFlowIntegrity.html">ControlFlowIntegrity</a> LLVM Control Flow Integrity (CFI) provides |
| forward-edge control flow protection.</li> |
| <li><a href="https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html">Hardware-assisted AddressSanitizer</a> a tool similar to |
| AddressSanitizer but based on partial hardware assistance.</li> |
| <li><a href="https://clang.llvm.org/docs/ControlFlowIntegrity.html#fsanitize-kcfi">KernelControlFlowIntegrity</a> LLVM Kernel Control Flow Integrity |
| (KCFI) provides forward-edge control flow protection for operating systems kernels.</li> |
| <li><a href="https://clang.llvm.org/docs/LeakSanitizer.html">LeakSanitizer</a> a run-time memory leak detector.</li> |
| <li><a href="https://clang.llvm.org/docs/MemorySanitizer.html">MemorySanitizer</a> a detector of uninitialized reads.</li> |
| <li><a href="https://clang.llvm.org/docs/ThreadSanitizer.html">ThreadSanitizer</a> a fast data race detector.</li> |
| </ul> |
| <h2 id="how-to-use-the-sanitizers"><a class="header" href="#how-to-use-the-sanitizers">How to use the sanitizers?</a></h2> |
| <p>To enable a sanitizer compile with <code>-Z sanitizer=...</code> option, where value is one |
| of <code>address</code>, <code>cfi</code>, <code>hwaddress</code>, <code>kcfi</code>, <code>leak</code>, <code>memory</code> or <code>thread</code>. |
| For more details on how to use sanitizers, |
| please refer to the sanitizer flag in <a href="https://doc.rust-lang.org/unstable-book">The Unstable Book</a>.</p> |
| <h2 id="how-are-sanitizers-implemented-in-rustc"><a class="header" href="#how-are-sanitizers-implemented-in-rustc">How are sanitizers implemented in rustc?</a></h2> |
| <p>The implementation of sanitizers (except CFI) relies almost entirely on LLVM. |
| The rustc is an integration point for LLVM compile time instrumentation passes |
| and runtime libraries. |
| Highlight of the most important aspects of the implementation:</p> |
| <ul> |
| <li> |
| <p>The sanitizer runtime libraries are part of the <a href="https://github.com/llvm/llvm-project/tree/main/compiler-rt">compiler-rt</a> project, and |
| <a href="https://github.com/rust-lang/rust/blob/1ead4761e9e2f056385768614c23ffa7acb6a19e/src/bootstrap/src/core/build_steps/llvm.rs#L958-L1031">will be built</a> on <a href="https://github.com/rust-lang/rust/blob/1ead4761e9e2f056385768614c23ffa7acb6a19e/src/bootstrap/src/core/build_steps/llvm.rs#L1073-L1111">supported targets</a> |
| when enabled in <code>bootstrap.toml</code>:</p> |
| <pre><code class="language-toml">[build] |
| sanitizers = true |
| </code></pre> |
| <p>The runtimes are <a href="https://github.com/rust-lang/rust/blob/1ead4761e9e2f056385768614c23ffa7acb6a19e/src/bootstrap/src/core/build_steps/compile.rs#L637-L676">placed into target libdir</a>.</p> |
| </li> |
| <li> |
| <p>During LLVM code generation, the functions intended for instrumentation are |
| <a href="https://github.com/rust-lang/rust/blob/1.55.0/compiler/rustc_codegen_llvm/src/attributes.rs#L42-L58">marked</a> with appropriate LLVM attribute: |
| <code>SanitizeAddress</code>, <code>SanitizeHWAddress</code>, <code>SanitizeMemory</code>, or <code>SanitizeThread</code>. |
| By default, all functions are instrumented, but this |
| behaviour can be changed with <code>#[sanitize(xyz = "on|off|<other>")]</code>.</p> |
| </li> |
| <li> |
| <p>The decision whether to perform instrumentation or not is possible only at a |
| function granularity. |
| In the cases were those decision differ between |
| functions, it might be necessary to inhibit inlining, both at <a href="https://github.com/rust-lang/rust/blob/1.55.0/compiler/rustc_mir/src/transform/inline.rs#L314-L316">MIR |
| level</a> and <a href="https://github.com/rust-lang/llvm-project/blob/9330ec5a4c1df5fc1fa62f993ed6a04da68cb040/llvm/include/llvm/IR/Attributes.td#L225-L241">LLVM level</a>.</p> |
| </li> |
| <li> |
| <p>The LLVM IR generated by rustc is instrumented by <a href="https://github.com/rust-lang/rust/blob/1.55.0/compiler/rustc_codegen_llvm/src/back/write.rs#L660-L678">dedicated LLVM |
| passes</a>, different for each sanitizer. |
| Instrumentation passes are invoked after optimization passes.</p> |
| </li> |
| <li> |
| <p>When producing an executable, the sanitizer specific runtime library is |
| <a href="https://github.com/rust-lang/rust/blob/1.55.0/compiler/rustc_codegen_ssa/src/back/link.rs#L1053-L1089">linked in</a>. |
| The libraries are searched for in the target libdir. |
| First, the search is relative to the overridden system root, and subsequently, |
| it is relative to the default system root. |
| Fall-back to the default system root |
| ensures that sanitizer runtimes remain available when using sysroot overrides |
| constructed by cargo <code>-Z build-std</code> or xargo.</p> |
| </li> |
| </ul> |
| <h2 id="testing-sanitizers"><a class="header" href="#testing-sanitizers">Testing sanitizers</a></h2> |
| <p>Sanitizers are validated by code generation tests in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/codegen-llvm"><code>tests/codegen-llvm/sanitize*.rs</code></a> and end-to-end functional tests in |
| <a href="https://github.com/rust-lang/rust/tree/HEAD/tests/ui/sanitizer"><code>tests/ui/sanitizer/</code></a> directory.</p> |
| <p>Testing sanitizer functionality requires the sanitizer runtimes (built when |
| <code>sanitizer = true</code> in <code>bootstrap.toml</code>) and target providing support for particular a sanitizer. |
| When a sanitizer is unsupported on a given target, sanitizer tests will be ignored. |
| This behaviour is controlled by compiletest <code>needs-sanitizer-*</code> directives.</p> |
| <h2 id="enabling-a-sanitizer-on-a-new-target"><a class="header" href="#enabling-a-sanitizer-on-a-new-target">Enabling a sanitizer on a new target</a></h2> |
| <p>To enable a sanitizer on a new target which is already supported by LLVM:</p> |
| <ol> |
| <li>Include the sanitizer in the list of <code>supported_sanitizers</code> in <a href="https://github.com/rust-lang/rust/blob/1.55.0/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs#L10-L11">the target |
| definition</a>. |
| <code>rustc --target .. -Zsanitizer=..</code> should now recognize the sanitizer as supported.</li> |
| <li><a href="https://github.com/rust-lang/rust/blob/1ead4761e9e2f056385768614c23ffa7acb6a19e/src/bootstrap/src/core/build_steps/llvm.rs#L1073-L1111">Build the runtime for the target and include it in the libdir.</a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/1.55.0/src/tools/compiletest/src/util.rs#L87-L116">Teach compiletest that your target now supports the sanitizer.</a> |
| Tests marked with <code>needs-sanitizer-*</code> should now run on the target.</li> |
| <li>Run tests <code>./x test --force-rerun tests/ui/sanitize/</code> to verify.</li> |
| <li><a href="https://github.com/rust-lang/rust/blob/1.55.0/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile#L94">--enable-sanitizers in the CI configuration</a> to build and |
| distribute the sanitizer runtime as part of the release process.</li> |
| </ol> |
| <h2 id="additional-information-1"><a class="header" href="#additional-information-1">Additional Information</a></h2> |
| <ul> |
| <li><a href="https://github.com/google/sanitizers/wiki/">Sanitizers project page</a></li> |
| <li><a href="https://clang.llvm.org/docs/AddressSanitizer.html">AddressSanitizer in Clang</a></li> |
| <li><a href="https://clang.llvm.org/docs/ControlFlowIntegrity.html">ControlFlowIntegrity in Clang</a></li> |
| <li><a href="https://clang.llvm.org/docs/HardwareAssistedAddressSanitizerDesign.html">Hardware-assisted AddressSanitizer</a></li> |
| <li><a href="https://clang.llvm.org/docs/ControlFlowIntegrity.html#fsanitize-kcfi">KernelControlFlowIntegrity in Clang</a></li> |
| <li><a href="https://clang.llvm.org/docs/LeakSanitizer.html">LeakSanitizer in Clang</a></li> |
| <li><a href="https://clang.llvm.org/docs/MemorySanitizer.html">MemorySanitizer in Clang</a></li> |
| <li><a href="https://clang.llvm.org/docs/ThreadSanitizer.html">ThreadSanitizer in Clang</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="debugging-support-in-the-rust-compiler"><a class="header" href="#debugging-support-in-the-rust-compiler">Debugging support in the Rust compiler</a></h1> |
| <p>This document explains the state of debugging tools support in the Rust compiler (rustc). |
| It gives an overview of GDB, LLDB, WinDbg/CDB, |
| as well as infrastructure around Rust compiler to debug Rust code. |
| If you want to learn how to debug the Rust compiler itself, |
| see <a href="compiler-debugging.html">Debugging the Compiler</a>.</p> |
| <p>The material is gathered from the video, |
| <a href="https://www.youtube.com/watch?v=elBxMRSNYr4">Tom Tromey discusses debugging support in rustc</a>.</p> |
| <h2 id="preliminaries"><a class="header" href="#preliminaries">Preliminaries</a></h2> |
| <h3 id="debuggers"><a class="header" href="#debuggers">Debuggers</a></h3> |
| <p>According to Wikipedia</p> |
| <blockquote> |
| <p>A <a href="https://en.wikipedia.org/wiki/Debugger">debugger or debugging tool</a> is a computer program that is used to test and debug |
| other programs (the "target" program).</p> |
| </blockquote> |
| <p>Writing a debugger from scratch for a language requires a lot of work, especially if |
| debuggers have to be supported on various platforms. GDB and LLDB, however, can be |
| extended to support debugging a language. This is the path that Rust has chosen. |
| This document's main goal is to document the said debuggers support in Rust compiler.</p> |
| <h3 id="dwarf"><a class="header" href="#dwarf">DWARF</a></h3> |
| <p>According to the <a href="http://dwarfstd.org">DWARF</a> standard website</p> |
| <blockquote> |
| <p>DWARF is a debugging file format used by many compilers and debuggers to support source level |
| debugging. It addresses the requirements of a number of procedural languages, |
| such as C, C++, and Fortran, and is designed to be extensible to other languages. |
| DWARF is architecture independent and applicable to any processor or operating system. |
| It is widely used on Unix, Linux and other operating systems, |
| as well as in stand-alone environments.</p> |
| </blockquote> |
| <p>DWARF reader is a program that consumes the DWARF format and creates debugger compatible output. |
| This program may live in the compiler itself. DWARF uses a data structure called |
| Debugging Information Entry (DIE) which stores the information as "tags" to denote functions, |
| variables etc., e.g., <code>DW_TAG_variable</code>, <code>DW_TAG_pointer_type</code>, <code>DW_TAG_subprogram</code> etc. |
| You can also invent your own tags and attributes.</p> |
| <h3 id="codeviewpdb"><a class="header" href="#codeviewpdb">CodeView/PDB</a></h3> |
| <p><a href="https://llvm.org/docs/PDB/index.html">PDB</a> (Program Database) is a file format created by Microsoft that contains debug information. |
| PDBs can be consumed by debuggers such as WinDbg/CDB and other tools to display debug information. |
| A PDB contains multiple streams that describe debug information about a specific binary such |
| as types, symbols, and source files used to compile the given binary. CodeView is another |
| format which defines the structure of <a href="https://llvm.org/docs/PDB/CodeViewSymbols.html">symbol records</a> and <a href="https://llvm.org/docs/PDB/CodeViewTypes.html">type records</a> that appear within |
| PDB streams.</p> |
| <h2 id="supported-debuggers"><a class="header" href="#supported-debuggers">Supported debuggers</a></h2> |
| <h3 id="gdb"><a class="header" href="#gdb">GDB</a></h3> |
| <h4 id="rust-expression-parser"><a class="header" href="#rust-expression-parser">Rust expression parser</a></h4> |
| <p>To be able to show debug output, we need an expression parser. |
| This (GDB) expression parser is written in <a href="https://www.gnu.org/software/bison/">Bison</a>, |
| and can parse only a subset of Rust expressions. |
| GDB parser was written from scratch and has no relation to any other parser, |
| including that of rustc.</p> |
| <p>GDB has Rust-like value and type output. It can print values and types in a way |
| that look like Rust syntax in the output. Or when you print a type as <a href="https://ftp.gnu.org/old-gnu/Manuals/gdb/html_node/gdb_109.html">ptype</a> in GDB, |
| it also looks like Rust source code. Checkout the documentation in the <a href="https://sourceware.org/gdb/onlinedocs/gdb/Rust.html">manual for GDB/Rust</a>.</p> |
| <h4 id="parser-extensions"><a class="header" href="#parser-extensions">Parser extensions</a></h4> |
| <p>Expression parser has a couple of extensions in it to facilitate features that you cannot do |
| with Rust. Some limitations are listed in the <a href="https://sourceware.org/gdb/onlinedocs/gdb/Rust.html">manual for GDB/Rust</a>. There is some special |
| code in the DWARF reader in GDB to support the extensions.</p> |
| <p>A couple of examples of DWARF reader support needed are as follows:</p> |
| <ol> |
| <li> |
| <p>Enum: Needed for support for enum types. |
| The Rust compiler writes the information about enum into DWARF, |
| and GDB reads the DWARF to understand where is the tag field, |
| or if there is a tag field, |
| or if the tag slot is shared with non-zero optimization etc.</p> |
| </li> |
| <li> |
| <p>Dissect trait objects: DWARF extension where the trait object's description in the DWARF |
| also points to a stub description of the corresponding vtable which in turn points to the |
| concrete type for which this trait object exists. This means that you can do a <code>print *object</code> |
| for that trait object, and GDB will understand how to find the correct type of the payload in |
| the trait object.</p> |
| </li> |
| </ol> |
| <p><strong>TODO</strong>: Figure out if the following should be mentioned in the GDB-Rust document rather than |
| this guide page so there is no duplication. This is regarding the following comments:</p> |
| <p><a href="https://github.com/rust-lang/rustc-dev-guide/pull/316#discussion_r284027340">This comment by Tom</a></p> |
| <blockquote> |
| <p>gdb's Rust extensions and limitations are documented in the gdb manual: |
| https://sourceware.org/gdb/onlinedocs/gdb/Rust.html -- however, this neglects to mention that |
| gdb convenience variables and registers follow the gdb $ convention, and that the Rust parser |
| implements the gdb @ extension.</p> |
| </blockquote> |
| <p><a href="https://github.com/rust-lang/rustc-dev-guide/pull/316#discussion_r285401353">This question by Aman</a></p> |
| <blockquote> |
| <p>@tromey do you think we should mention this part in the GDB-Rust document rather than this |
| document so there is no duplication etc.?</p> |
| </blockquote> |
| <h3 id="lldb"><a class="header" href="#lldb">LLDB</a></h3> |
| <h4 id="rust-expression-parser-1"><a class="header" href="#rust-expression-parser-1">Rust expression parser</a></h4> |
| <p>This expression parser is written in C++. It is a type of <a href="https://en.wikipedia.org/wiki/Recursive_descent_parser">Recursive Descent parser</a>. |
| It implements slightly less of the Rust language than GDB. |
| LLDB has Rust-like value and type output.</p> |
| <h4 id="developer-notes"><a class="header" href="#developer-notes">Developer notes</a></h4> |
| <ul> |
| <li>LLDB has a plugin architecture but that does not work for language support.</li> |
| <li>GDB generally works better on Linux.</li> |
| </ul> |
| <h3 id="windbgcdb"><a class="header" href="#windbgcdb">WinDbg/CDB</a></h3> |
| <p>Microsoft provides <a href="https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/">Windows Debugging Tools</a> such as the Windows Debugger (WinDbg) and |
| the Console Debugger (CDB) which both support debugging programs written in Rust. These |
| debuggers parse the debug info for a binary from the <code>PDB</code>, if available, to construct a |
| visualization to serve up in the debugger.</p> |
| <h4 id="natvis"><a class="header" href="#natvis">Natvis</a></h4> |
| <p>Both WinDbg and CDB support defining and viewing custom visualizations for any given type |
| within the debugger using the Natvis framework. The Rust compiler defines a set of Natvis |
| files that define custom visualizations for a subset of types in the standard libraries such |
| as, <code>std</code>, <code>core</code>, and <code>alloc</code>. These Natvis files are embedded into <code>PDBs</code> generated by the |
| <code>*-pc-windows-msvc</code> target triples to automatically enable these custom visualizations when |
| debugging. This default can be overridden by setting the <code>strip</code> rustc flag to either <code>debuginfo</code> |
| or <code>symbols</code>.</p> |
| <p>Rust has support for embedding Natvis files for crates outside of the standard libraries by |
| using the <code>#[debugger_visualizer]</code> attribute. |
| For more details on how to embed debugger visualizers, |
| please refer to the section on the <a href="https://doc.rust-lang.org/nightly/reference/attributes/debugger.html#the-debugger_visualizer-attribute"><code>debugger_visualizer</code> attribute</a>.</p> |
| <h2 id="dwarf-and-rustc"><a class="header" href="#dwarf-and-rustc">DWARF and <code>rustc</code></a></h2> |
| <p><a href="http://dwarfstd.org">DWARF</a> is the standard way compilers generate debugging information that debuggers read. |
| It is <em>the</em> debugging format on macOS and Linux. |
| It is a multi-language and extensible format, |
| and is mostly good enough for Rust's purposes. |
| Hence, the current implementation reuses DWARF's concepts. |
| This is true even if some of the concepts in DWARF do not align with Rust semantically because, |
| generally, there can be some kind of mapping between the two.</p> |
| <p>We have some DWARF extensions that the Rust compiler emits and the debuggers understand that |
| are <em>not</em> in the DWARF standard.</p> |
| <ul> |
| <li> |
| <p>Rust compiler will emit DWARF for a virtual table, and this <code>vtable</code> object will have a |
| <code>DW_AT_containing_type</code> that points to the real type. This lets debuggers dissect a trait object |
| pointer to correctly find the payload. E.g., here's such a DIE, from a test case in the gdb |
| repository:</p> |
| <pre><code class="language-asm"><1><1a9>: Abbrev Number: 3 (DW_TAG_structure_type) |
| <1aa> DW_AT_containing_type: <0x1b4> |
| <1ae> DW_AT_name : (indirect string, offset: 0x23d): vtable |
| <1b2> DW_AT_byte_size : 0 |
| <1b3> DW_AT_alignment : 8 |
| </code></pre> |
| </li> |
| <li> |
| <p>The other extension is that the Rust compiler can emit a tagless discriminated union. |
| See <a href="http://dwarfstd.org/ShowIssue.php?issue=180517.2">DWARF feature request</a> for this item.</p> |
| </li> |
| </ul> |
| <h3 id="current-limitations-of-dwarf"><a class="header" href="#current-limitations-of-dwarf">Current limitations of DWARF</a></h3> |
| <ul> |
| <li>Traits - require a bigger change than normal to DWARF, on how to represent Traits in DWARF.</li> |
| <li>DWARF provides no way to differentiate between Structs and Tuples. Rust compiler emits |
| fields with <code>__0</code> and debuggers look for a sequence of such names to overcome this limitation. |
| For example, in this case the debugger would look at a field via <code>x.__0</code> instead of <code>x.0</code>. |
| This is resolved via the Rust parser in the debugger so now you can do <code>x.0</code>.</li> |
| </ul> |
| <p>DWARF relies on debuggers to know some information about platform ABI. |
| Rust does not do that all the time.</p> |
| <h2 id="developer-notes-1"><a class="header" href="#developer-notes-1">Developer notes</a></h2> |
| <p>This section is from the talk about certain aspects of development.</p> |
| <h2 id="what-is-missing"><a class="header" href="#what-is-missing">What is missing</a></h2> |
| <h3 id="code-signing-for-lldb-debug-server-on-macos"><a class="header" href="#code-signing-for-lldb-debug-server-on-macos">Code signing for LLDB debug server on macOS</a></h3> |
| <p>According to Wikipedia, <a href="https://en.wikipedia.org/wiki/System_Integrity_Protection">System Integrity Protection</a> is</p> |
| <blockquote> |
| <p>System Integrity Protection (SIP, sometimes referred to as rootless) is a security feature |
| of Apple's macOS operating system introduced in OS X El Capitan. It comprises a number of |
| mechanisms that are enforced by the kernel. A centerpiece is the protection of system-owned |
| files and directories against modifications by processes without a specific "entitlement", |
| even when executed by the root user or a user with root privileges (sudo).</p> |
| </blockquote> |
| <p>It prevents processes using <code>ptrace</code> syscall. If a process wants to use <code>ptrace</code> it has to be |
| code signed. The certificate that signs it has to be trusted on your machine.</p> |
| <p>See <a href="https://developer.apple.com/library/archive/releasenotes/MacOSX/WhatsNewInOSX/Articles/MacOSX10_11.html#//apple_ref/doc/uid/TP40016227-SW11">Apple developer documentation for System Integrity Protection</a>.</p> |
| <p>We may need to sign up with Apple and get the keys to do this signing. Tom has looked into if |
| Mozilla cannot do this because it is at the maximum number of |
| keys it is allowed to sign. Tom does not know if Mozilla could get more keys.</p> |
| <p>Alternatively, Tom suggests that maybe a Rust legal entity is needed to get the keys via Apple. |
| This problem is not technical in nature. If we had such a key we could sign GDB as well and |
| ship that.</p> |
| <h3 id="dwarf-and-traits"><a class="header" href="#dwarf-and-traits">DWARF and Traits</a></h3> |
| <p>Rust traits are not emitted into DWARF at all. The impact of this is calling a method <code>x.method()</code> |
| does not work as is. The reason being that method is implemented by a trait, as opposed |
| to a type. That information is not present so finding trait methods is missing.</p> |
| <p>DWARF has a notion of interface types (possibly added for Java). Tom's idea was to use this |
| interface type as traits.</p> |
| <p>DWARF only deals with concrete names, not the reference types. So, a given implementation of a |
| trait for a type would be one of these interfaces (<code>DW_tag_interface</code> type). Also, the type for |
| which it is implemented would describe all the interfaces this type implements. This requires a |
| DWARF extension.</p> |
| <p>Issue on Github: <a href="https://github.com/rust-lang/rust/issues/33014">https://github.com/rust-lang/rust/issues/33014</a></p> |
| <h2 id="typical-process-for-a-debug-info-change-llvm"><a class="header" href="#typical-process-for-a-debug-info-change-llvm">Typical process for a Debug Info change (LLVM)</a></h2> |
| <p>LLVM has Debug Info (DI) builders. This is the primary thing that Rust calls into. |
| This is why we need to change LLVM first because that is emitted first and not DWARF directly. |
| This is a kind of metadata that you construct and hand-off to LLVM. For the Rustc/LLVM hand-off |
| some LLVM DI builder methods are called to construct representation of a type.</p> |
| <p>The steps of this process are as follows:</p> |
| <ol> |
| <li> |
| <p>LLVM needs changing.</p> |
| <p>LLVM does not emit Interface types at all, so this needs to be implemented in the LLVM first.</p> |
| <p>Get sign off on LLVM maintainers that this is a good idea.</p> |
| </li> |
| <li> |
| <p>Change the DWARF extension.</p> |
| </li> |
| <li> |
| <p>Update the debuggers.</p> |
| <p>Update DWARF readers, expression evaluators.</p> |
| </li> |
| <li> |
| <p>Update Rust compiler.</p> |
| <p>Change it to emit this new information.</p> |
| </li> |
| </ol> |
| <h3 id="procedural-macro-stepping"><a class="header" href="#procedural-macro-stepping">Procedural macro stepping</a></h3> |
| <p>A deeply profound question is that how do you actually debug a procedural macro? |
| What is the location you emit for a macro expansion? Consider some of the following cases -</p> |
| <ul> |
| <li>You can emit location of the invocation of the macro.</li> |
| <li>You can emit the location of the definition of the macro.</li> |
| <li>You can emit locations of the content of the macro.</li> |
| </ul> |
| <p>RFC: <a href="https://github.com/rust-lang/rfcs/pull/2117">https://github.com/rust-lang/rfcs/pull/2117</a></p> |
| <p>Focus is to let macros decide what to do. This can be achieved by having some kind of attribute |
| that lets the macro tell the compiler where the line marker should be. This affects where you |
| set the breakpoints and what happens when you step it.</p> |
| <h2 id="source-file-checksums-in-debug-info"><a class="header" href="#source-file-checksums-in-debug-info">Source file checksums in debug info</a></h2> |
| <p>Both DWARF and CodeView (PDB) support embedding a cryptographic hash of each source file that |
| contributed to the associated binary.</p> |
| <p>The cryptographic hash can be used by a debugger to verify that the source file matches the |
| executable. If the source file does not match, the debugger can provide a warning to the user.</p> |
| <p>The hash can also be used to prove that a given source file has not been modified since it was |
| used to compile an executable. Because MD5 and SHA1 both have demonstrated vulnerabilities, |
| using SHA256 is recommended for this application.</p> |
| <p>The Rust compiler stores the hash for each source file in the corresponding <code>SourceFile</code> in |
| the <code>SourceMap</code>. The hashes of input files to external crates are stored in <code>rlib</code> metadata.</p> |
| <p>A default hashing algorithm is set in the target specification. This allows the target to |
| specify the best hash available, since not all targets support all hash algorithms.</p> |
| <p>The hashing algorithm for a target can also be overridden with the <code>-Z source-file-checksum=</code> |
| command-line option.</p> |
| <h4 id="dwarf-5"><a class="header" href="#dwarf-5">DWARF 5</a></h4> |
| <p>DWARF version 5 supports embedding an MD5 hash to validate the source file version in use. |
| DWARF 5 - Section 6.2.4.1 opcode DW_LNCT_MD5</p> |
| <h4 id="llvm"><a class="header" href="#llvm">LLVM</a></h4> |
| <p>LLVM IR supports MD5 and SHA1 (and SHA256 in LLVM 11+) source file checksums in the DIFile node.</p> |
| <p><a href="https://llvm.org/docs/LangRef.html#difile">LLVM DIFile documentation</a></p> |
| <h4 id="microsoft-visual-c-compiler-zh-option"><a class="header" href="#microsoft-visual-c-compiler-zh-option">Microsoft Visual C++ Compiler /ZH option</a></h4> |
| <p>The MSVC compiler supports embedding MD5, SHA1, or SHA256 hashes in the PDB using the <code>/ZH</code> |
| compiler option.</p> |
| <p><a href="https://docs.microsoft.com/en-us/cpp/build/reference/zh">MSVC /ZH documentation</a></p> |
| <h4 id="clang"><a class="header" href="#clang">Clang</a></h4> |
| <p>Clang always embeds an MD5 checksum, though this does not appear in documentation.</p> |
| <h2 id="future-work-1"><a class="header" href="#future-work-1">Future work</a></h2> |
| <h4 id="name-mangling-changes"><a class="header" href="#name-mangling-changes">Name mangling changes</a></h4> |
| <ul> |
| <li>New demangler in <code>libiberty</code> (gcc source tree).</li> |
| <li>New demangler in LLVM or LLDB.</li> |
| </ul> |
| <p><strong>TODO</strong>: Check the location of the demangler source. <a href="https://github.com/rust-lang/rustc-dev-guide/issues/1157">#1157</a></p> |
| <h4 id="reuse-rust-compiler-for-expressions"><a class="header" href="#reuse-rust-compiler-for-expressions">Reuse Rust compiler for expressions</a></h4> |
| <p>This is an important idea because debuggers by and large do not try to implement type |
| inference. You need to be much more explicit when you type into the debugger than your |
| actual source code. So, you cannot just copy and paste an expression from your source |
| code to debugger and expect the same answer but this would be nice. This can be helped |
| by using compiler.</p> |
| <p>It is certainly doable but it is a large project. You certainly need a bridge to the |
| debugger because the debugger alone has access to the memory. Both GDB (gcc) and LLDB (clang) |
| have this feature. LLDB uses Clang to compile code to JIT and GDB can do the same with GCC.</p> |
| <p>Both debuggers expression evaluation implement both a superset and a subset of Rust. |
| They implement just the expression language, |
| but they also add some extensions like GDB has convenience variables. |
| Therefore, if you are taking this route, |
| then you not only need to do this bridge, |
| but may have to add some mode to let the compiler understand some extensions.</p> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="background-topics"><a class="header" href="#background-topics">Background topics</a></h1> |
| <p>This section covers a numbers of common compiler terms that arise in |
| this guide. We try to give the general definition while providing some |
| Rust-specific context.</p> |
| <p><a id="cfg"></a></p> |
| <h2 id="what-is-a-control-flow-graph"><a class="header" href="#what-is-a-control-flow-graph">What is a control-flow graph?</a></h2> |
| <p>A control-flow graph (CFG) is a common term from compilers. If you've ever |
| used a flow-chart, then the concept of a control-flow graph will be |
| pretty familiar to you. It's a representation of your program that |
| clearly exposes the underlying control flow.</p> |
| <p>A control-flow graph is structured as a set of <strong>basic blocks</strong> |
| connected by edges. The key idea of a basic block is that it is a set |
| of statements that execute "together" – that is, whenever you branch |
| to a basic block, you start at the first statement and then execute |
| all the remainder. Only at the end of the block is there the |
| possibility of branching to more than one place (in MIR, we call that |
| final statement the <strong>terminator</strong>):</p> |
| <pre><code class="language-mir">bb0: { |
| statement0; |
| statement1; |
| statement2; |
| ... |
| terminator; |
| } |
| </code></pre> |
| <p>Many expressions that you are used to in Rust compile down to multiple |
| basic blocks. For example, consider an if statement:</p> |
| <pre><code class="language-rust ignore">a = 1; |
| if some_variable { |
| b = 1; |
| } else { |
| c = 1; |
| } |
| d = 1;</code></pre> |
| <p>This would compile into four basic blocks in MIR. In textual form, it looks like |
| this:</p> |
| <pre><code class="language-mir">BB0: { |
| a = 1; |
| if some_variable { |
| goto BB1; |
| } else { |
| goto BB2; |
| } |
| } |
| |
| BB1: { |
| b = 1; |
| goto BB3; |
| } |
| |
| BB2: { |
| c = 1; |
| goto BB3; |
| } |
| |
| BB3: { |
| d = 1; |
| ... |
| } |
| </code></pre> |
| <p>In graphical form, it looks like this:</p> |
| <pre><code> BB0 |
| +--------------------+ |
| | a = 1; | |
| +--------------------+ |
| / \ |
| if some_variable else |
| / \ |
| BB1 / \ BB2 |
| +-----------+ +-----------+ |
| | b = 1; | | c = 1; | |
| +-----------+ +-----------+ |
| \ / |
| \ / |
| \ BB3 / |
| +----------+ |
| | d = 1; | |
| | ... | |
| +----------+ |
| </code></pre> |
| <p>When using a control-flow graph, a loop simply appears as a cycle in |
| the graph, and the <code>break</code> keyword translates into a path out of that |
| cycle.</p> |
| <p><a id="dataflow"></a></p> |
| <h2 id="what-is-a-dataflow-analysis"><a class="header" href="#what-is-a-dataflow-analysis">What is a dataflow analysis?</a></h2> |
| <p><a href="https://cs.au.dk/~amoeller/spa/"><em>Static Program Analysis</em></a> by Anders Møller |
| and Michael I. Schwartzbach is an incredible resource!</p> |
| <p><em>Dataflow analysis</em> is a type of static analysis that is common in many |
| compilers. It describes a general technique, rather than a particular analysis.</p> |
| <p>The basic idea is that we can walk over a <a href="appendix/background.html#cfg">control-flow graph (CFG)</a> and |
| keep track of what some value could be. At the end of the walk, we might have |
| shown that some claim is true or not necessarily true (e.g. "this variable must |
| be initialized"). <code>rustc</code> tends to do dataflow analyses over the MIR, since MIR |
| is already a CFG.</p> |
| <p>For example, suppose we want to check that <code>x</code> is initialized before it is used |
| in this snippet:</p> |
| <pre><code class="language-rust ignore">fn foo() { |
| let mut x; |
| |
| if some_cond { |
| x = 1; |
| } |
| |
| dbg!(x); |
| }</code></pre> |
| <p>A CFG for this code might look like this:</p> |
| <pre><code class="language-txt"> +------+ |
| | Init | (A) |
| +------+ |
| | \ |
| | if some_cond |
| else \ +-------+ |
| | \| x = 1 | (B) |
| | +-------+ |
| | / |
| +---------+ |
| | dbg!(x) | (C) |
| +---------+ |
| </code></pre> |
| <p>We can do the dataflow analysis as follows: we will start off with a flag <code>init</code> |
| which indicates if we know <code>x</code> is initialized. As we walk the CFG, we will |
| update the flag. At the end, we can check its value.</p> |
| <p>So first, in block (A), the variable <code>x</code> is declared but not initialized, so |
| <code>init = false</code>. In block (B), we initialize the value, so we know that <code>x</code> is |
| initialized. So at the end of (B), <code>init = true</code>.</p> |
| <p>Block (C) is where things get interesting. Notice that there are two incoming |
| edges, one from (A) and one from (B), corresponding to whether <code>some_cond</code> is true or not. |
| But we cannot know that! It could be the case the <code>some_cond</code> is always true, |
| so that <code>x</code> is actually always initialized. It could also be the case that |
| <code>some_cond</code> depends on something random (e.g. the time), so <code>x</code> may not be |
| initialized. In general, we cannot know statically (due to <a href="https://en.wikipedia.org/wiki/Rice%27s_theorem">Rice's |
| Theorem</a>). So what should the value of <code>init</code> be in block (C)?</p> |
| <p>Generally, in dataflow analyses, if a block has multiple parents (like (C) in |
| our example), its dataflow value will be some function of all its parents (and |
| of course, what happens in (C)). Which function we use depends on the analysis |
| we are doing.</p> |
| <p>In this case, we want to be able to prove definitively that <code>x</code> must be |
| initialized before use. This forces us to be conservative and assume that |
| <code>some_cond</code> might be false sometimes. So our "merging function" is "and". That |
| is, <code>init = true</code> in (C) if <code>init = true</code> in (A) <em>and</em> in (B) (or if <code>x</code> is |
| initialized in (C)). But this is not the case; in particular, <code>init = false</code> in |
| (A), and <code>x</code> is not initialized in (C). Thus, <code>init = false</code> in (C); we can |
| report an error that "<code>x</code> may not be initialized before use".</p> |
| <p>There is definitely a lot more that can be said about dataflow analyses. There is an |
| extensive body of research literature on the topic, including a lot of theory. |
| We only discussed a forwards analysis, but backwards dataflow analysis is also |
| useful. For example, rather than starting from block (A) and moving forwards, |
| we might have started with the usage of <code>x</code> and moved backwards to try to find |
| its initialization.</p> |
| <p><a id="quantified"></a></p> |
| <h2 id="what-is-universally-quantified-what-about-existentially-quantified"><a class="header" href="#what-is-universally-quantified-what-about-existentially-quantified">What is "universally quantified"? What about "existentially quantified"?</a></h2> |
| <p>In math, a predicate may be <em>universally quantified</em> or <em>existentially |
| quantified</em>:</p> |
| <ul> |
| <li><em>Universal</em> quantification: |
| <ul> |
| <li>the predicate holds if it is true for all possible inputs.</li> |
| <li>Traditional notation: ∀x: P(x). Read as "for all x, P(x) holds".</li> |
| </ul> |
| </li> |
| <li><em>Existential</em> quantification: |
| <ul> |
| <li>the predicate holds if there is any input where it is true, i.e., there |
| only has to be a single input.</li> |
| <li>Traditional notation: ∃x: P(x). Read as "there exists x such that P(x) holds".</li> |
| </ul> |
| </li> |
| </ul> |
| <p>In Rust, they come up in type checking and trait solving. For example,</p> |
| <pre><code class="language-rust ignore">fn foo<T>()</code></pre> |
| <p>This function claims that the function is well-typed for all types <code>T</code>: <code>∀ T: well_typed(foo)</code>.</p> |
| <p>Another example:</p> |
| <pre><code class="language-rust ignore">fn foo<'a>(_: &'a usize)</code></pre> |
| <p>This function claims that for any lifetime <code>'a</code> (determined by the |
| caller), it is well-typed: <code>∀ 'a: well_typed(foo)</code>.</p> |
| <p>Another example:</p> |
| <pre><code class="language-rust ignore">fn foo<F>() |
| where for<'a> F: Fn(&'a u8)</code></pre> |
| <p>This function claims that it is well-typed for all types <code>F</code> such that for all |
| lifetimes <code>'a</code>, <code>F: Fn(&'a u8)</code>: <code>∀ F: ∀ 'a: (F: Fn(&'a u8)) => well_typed(foo)</code>.</p> |
| <p>One more example:</p> |
| <pre><code class="language-rust ignore">fn foo(_: dyn Debug)</code></pre> |
| <p>This function claims that there exists some type <code>T</code> that implements <code>Debug</code> |
| such that the function is well-typed: <code>∃ T: (T: Debug) and well_typed(foo)</code>.</p> |
| <p><a id="variance"></a></p> |
| <h2 id="what-is-a-de-bruijn-index"><a class="header" href="#what-is-a-de-bruijn-index">What is a de Bruijn Index?</a></h2> |
| <p><a href="https://en.wikipedia.org/wiki/De_Bruijn_index">De Bruijn indices</a> are a way of representing, using only integers, |
| which variables are bound in which binders. They were originally invented for |
| use in lambda calculus evaluation (see <a href="https://en.wikipedia.org/wiki/De_Bruijn_index">this Wikipedia article</a> for |
| more). In <code>rustc</code>, we use de Bruijn indices to <a href="appendix/../ty_module/generic_arguments.html">represent generic types</a>.</p> |
| <p>Here is a basic example of how de Bruijn indices might be used for closures (we |
| don't actually do this in <code>rustc</code> though!):</p> |
| <pre><code class="language-rust ignore">|x| { |
| f(x) // de Bruijn index of `x` is 1 because `x` is bound 1 level up |
| |
| |y| { |
| g(x, y) // index of `x` is 2 because it is bound 2 levels up |
| // index of `y` is 1 because it is bound 1 level up |
| } |
| }</code></pre> |
| <h2 id="what-are-co--and-contra-variance"><a class="header" href="#what-are-co--and-contra-variance">What are co- and contra-variance?</a></h2> |
| <p>Check out the subtyping chapter from the |
| <a href="https://doc.rust-lang.org/nomicon/subtyping.html">Rust Nomicon</a>.</p> |
| <p>See the <a href="appendix/../variance.html">variance</a> chapter of this guide for more info on how |
| the type checker handles variance.</p> |
| <p><a id="free-vs-bound"></a></p> |
| <h2 id="what-is-a-free-region-or-a-free-variable-what-about-bound-region"><a class="header" href="#what-is-a-free-region-or-a-free-variable-what-about-bound-region">What is a "free region" or a "free variable"? What about "bound region"?</a></h2> |
| <p>Let's describe the concepts of free vs bound in terms of program |
| variables, since that's the thing we're most familiar with.</p> |
| <ul> |
| <li>Consider this expression, which creates a closure: <code>|a, b| a + b</code>. |
| Here, the <code>a</code> and <code>b</code> in <code>a + b</code> refer to the arguments that the closure will |
| be given when it is called. We say that the <code>a</code> and <code>b</code> there are <strong>bound</strong> to |
| the closure, and that the closure signature <code>|a, b|</code> is a <strong>binder</strong> for the |
| names <code>a</code> and <code>b</code> (because any references to <code>a</code> or <code>b</code> within refer to the |
| variables that it introduces).</li> |
| <li>Consider this expression: <code>a + b</code>. In this expression, <code>a</code> and <code>b</code> refer to |
| local variables that are defined <em>outside</em> of the expression. We say that |
| those variables <strong>appear free</strong> in the expression (i.e., they are <strong>free</strong>, |
| not <strong>bound</strong> (tied up)).</li> |
| </ul> |
| <p>So there you have it: a variable "appears free" in some |
| expression/statement/whatever if it refers to something defined |
| outside of that expressions/statement/whatever. Equivalently, we can |
| then refer to the "free variables" of an expression – which is just |
| the set of variables that "appear free".</p> |
| <p>So what does this have to do with regions? Well, we can apply the |
| analogous concept to type and regions. For example, in the type <code>&'a u32</code>, <code>'a</code> appears free. But in the type <code>for<'a> fn(&'a u32)</code>, it |
| does not.</p> |
| <h1 id="further-reading-about-compilers"><a class="header" href="#further-reading-about-compilers">Further Reading About Compilers</a></h1> |
| <blockquote> |
| <p>Thanks to <code>mem</code>, <code>scottmcm</code>, and <code>Levi</code> on the official Discord for the |
| recommendations, and to <code>tinaun</code> for posting a link to a <a href="https://web.archive.org/web/20181230012554/https://twitter.com/graydon_pub/status/1039615569132118016">twitter thread from |
| Graydon Hoare</a> |
| which had some more recommendations!</p> |
| <p>Other sources: https://gcc.gnu.org/wiki/ListOfCompilerBooks</p> |
| <p>If you have other suggestions, please feel free to open an issue or PR.</p> |
| </blockquote> |
| <h2 id="books"><a class="header" href="#books">Books</a></h2> |
| <ul> |
| <li><a href="https://www.cis.upenn.edu/~bcpierce/tapl/">Types and Programming Languages</a></li> |
| <li><a href="https://www.cs.rochester.edu/~scott/pragmatics/">Programming Language Pragmatics</a></li> |
| <li><a href="https://www.cs.cmu.edu/~rwh/pfpl/">Practical Foundations for Programming Languages</a></li> |
| <li><a href="https://www.pearson.com/us/higher-education/program/Aho-Compilers-Principles-Techniques-and-Tools-2nd-Edition/PGM167067.html">Compilers: Principles, Techniques, and Tools, 2nd Edition</a></li> |
| <li><a href="https://www.cs.kent.ac.uk/people/staff/rej/gcbook/">Garbage Collection: Algorithms for Automatic Dynamic Memory Management</a></li> |
| <li><a href="https://www.amazon.com/Linkers-Kaufmann-Software-Engineering-Programming/dp/1558604960">Linkers and Loaders</a> (There are also free versions of this, but the version we had linked seems to be offline at the moment.)</li> |
| <li><a href="https://www.goodreads.com/book/show/887908.Advanced_Compiler_Design_and_Implementation">Advanced Compiler Design and Implementation</a></li> |
| <li><a href="https://www.goodreads.com/book/show/2063103.Building_an_Optimizing_Compiler">Building an Optimizing Compiler</a></li> |
| <li><a href="http://www.craftinginterpreters.com/">Crafting Interpreters</a></li> |
| </ul> |
| <h2 id="courses"><a class="header" href="#courses">Courses</a></h2> |
| <ul> |
| <li><a href="https://www.cs.uoregon.edu/research/summerschool/archives.html">University of Oregon Programming Languages Summer School archive</a></li> |
| </ul> |
| <h2 id="wikis"><a class="header" href="#wikis">Wikis</a></h2> |
| <ul> |
| <li><a href="https://en.wikipedia.org/wiki/List_of_programming_languages_by_type">Wikipedia</a></li> |
| <li><a href="https://esolangs.org/wiki/Main_Page">Esoteric Programming Languages</a></li> |
| <li><a href="https://plato.stanford.edu/index.html">Stanford Encyclopedia of Philosophy</a></li> |
| <li><a href="https://ncatlab.org/nlab/show/HomePage">nLab</a></li> |
| </ul> |
| <h2 id="misc-papers-and-blog-posts"><a class="header" href="#misc-papers-and-blog-posts">Misc Papers and Blog Posts</a></h2> |
| <ul> |
| <li><a href="https://www.cse.chalmers.se/research/group/logic/book/">Programming in Martin-Löf's Type Theory</a></li> |
| <li><a href="https://dl.acm.org/doi/10.1145/3093333.3009882">Polymorphism, Subtyping, and Type Inference in MLsub</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="glossary"><a class="header" href="#glossary">Glossary</a></h1> |
| <div class="table-wrapper"><table><thead><tr><th>Term</th><th>Meaning</th></tr></thead><tbody> |
| <tr><td><span id="arena">arena, arena allocation</span></td><td>An <em>arena</em> is a large memory buffer from which other memory allocations are made. This style of allocation is called <em>arena allocation</em>. See <a href="appendix/../memory.html">this chapter</a> for more info.</td></tr> |
| <tr><td><span id="ast">AST</span></td><td>The <em>abstract syntax tree</em> produced by the <code>rustc_ast</code> crate; reflects user syntax very closely.</td></tr> |
| <tr><td><span id="apit">APIT</span></td><td>An argument-position <code>impl Trait</code>. Also known as an anonymous type parameter. (<a href="https://doc.rust-lang.org/reference/types/impl-trait.html#anonymous-type-parameters">see the reference</a>).</td></tr> |
| <tr><td><span id="binder">binder</span></td><td>A <em>binder</em> is a place where a variable or type is declared; for example, the <code><T></code> is a binder for the generic type parameter <code>T</code> in <code>fn foo<T>(..)</code>, and |<code>a</code>|<code> ...</code> is a binder for the parameter <code>a</code>. See <a href="appendix/./background.html#free-vs-bound">the background chapter for more</a>.</td></tr> |
| <tr><td><span id="body-id"><code>BodyId</code></span></td><td>An identifier that refers to a specific body (definition of a function or constant) in the crate. See <a href="appendix/../hir.html#identifiers-in-the-hir">the HIR chapter for more</a>.</td></tr> |
| <tr><td><span id="bound-var">bound variable</span></td><td>A <em>bound variable</em> is one that is declared within an expression/term. For example, the variable <code>a</code> is bound within the closure expression |<code>a</code>|<code> a * 2</code>. See <a href="appendix/./background.html#free-vs-bound">the background chapter for more</a></td></tr> |
| <tr><td><span id="codegen">codegen</span></td><td>Short for <em>code generation</em>. The code to translate MIR into LLVM IR.</td></tr> |
| <tr><td><span id="codegen-unit">codegen unit</span></td><td>When we produce LLVM IR, we group the Rust code into a number of codegen units (sometimes abbreviated as CGUs). Each of these units is processed by LLVM independently from one another, enabling parallelism. They are also the unit of incremental re-use. (<a href="appendix/../backend/codegen.html">see more</a>)</td></tr> |
| <tr><td><span id="completeness">completeness</span></td><td>A technical term in type theory, it means that every type-safe program also type-checks. Having both soundness and completeness is very hard, and usually soundness is more important. (see "soundness").</td></tr> |
| <tr><td><span id="cfg">control-flow graph</span></td><td>A representation of the control-flow of a program; see <a href="appendix/./background.html#cfg">the background chapter for more</a></td></tr> |
| <tr><td><span id="ctfe">CTFE</span></td><td>Short for <em>compile-time function evaluation</em>, this is the ability of the compiler to evaluate <code>const fn</code>s at compile time. This is part of the compiler's constant evaluation system. (<a href="appendix/../const-eval.html">see more</a>)</td></tr> |
| <tr><td><span id="cx"><code>cx</code></span></td><td>We tend to use <em>cx</em> as an abbreviation for <em>context</em>. See also <code>tcx</code>, <code>infcx</code>, etc.</td></tr> |
| <tr><td><span id="ctxt"><code>ctxt</code></span></td><td>We also use <em>ctxt</em> as an abbreviation for <em>context</em>, e.g. <a href="appendix/glossary.html#TyCtxt"><code>TyCtxt</code></a>. See also <a href="appendix/glossary.html#cx">cx</a> or <a href="appendix/glossary.html#tcx">tcx</a>.</td></tr> |
| <tr><td><span id="dag">DAG</span></td><td>A <em>directed acyclic graph</em> is used during compilation to keep track of dependencies between queries. (<a href="appendix/../queries/incremental-compilation.html">see more</a>)</td></tr> |
| <tr><td><span id="data-flow">data-flow analysis</span></td><td>A static analysis that figures out what properties are true at each point in the control-flow of a program; see <a href="appendix/./background.html#dataflow">the background chapter for more</a>.</td></tr> |
| <tr><td><span id="debruijn">de Bruijn index</span></td><td>A technique for describing which binder a variable is bound by using only integers. It has the benefit that it is invariant under variable renaming. (<a href="appendix/./background.html#what-is-a-debruijn-index">see more</a>)</td></tr> |
| <tr><td><span id="def-id"><code>DefId</code></span></td><td>An index identifying a definition (see <code>rustc_middle/src/hir/def_id.rs</code>). Uniquely identifies a <code>DefPath</code>. See <a href="appendix/../hir.html#identifiers-in-the-hir">the HIR chapter for more</a>.</td></tr> |
| <tr><td><span id="discriminant">discriminant</span></td><td>The underlying value associated with an enum variant or generator state to indicate it as "active" (but not to be confused with its <a href="appendix/glossary.html#variant-idx">"variant index"</a>). At runtime, the discriminant of the active variant is encoded in the <a href="appendix/glossary.html#tag">tag</a>.</td></tr> |
| <tr><td><span id="double-ptr">double pointer</span></td><td>A pointer with additional metadata. See <a href="appendix/glossary.html#fat-ptr">fat pointer</a> for more.</td></tr> |
| <tr><td><span id="drop-glue">drop glue</span></td><td>(Internal) compiler-generated instructions that handle calling the destructors (<code>Drop</code>) for data types.</td></tr> |
| <tr><td><span id="dst">DST</span></td><td>Short for <em>dynamically-sized type</em>, this is a type for which the compiler cannot statically know the size in memory (e.g. <code>str</code> or <code>[u8]</code>). Such types don't implement <code>Sized</code> and cannot be allocated on the stack. They can only occur as the last field in a struct. They can only be used behind a pointer (e.g. <code>&str</code> or <code>&[u8]</code>).</td></tr> |
| <tr><td><span id="ebl">early-bound lifetime</span></td><td>A lifetime region that is substituted at its definition site. Bound in an item's <code>Generics</code> and substituted/instantiated using a <code>GenericArgs</code>. Contrast with <strong>late-bound lifetime</strong>. (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/region_kind/enum.RegionKind.html#bound-regions">see more</a>)</td></tr> |
| <tr><td><span id="effect">effects</span></td><td>Right now only means const traits and <code>~const</code> bounds. (<a href="appendix/../effects.html">see more</a>)</td></tr> |
| <tr><td><span id="empty-type">empty type</span></td><td>See <a href="appendix/glossary.html#ut">uninhabited type</a>.</td></tr> |
| <tr><td><span id="fat-ptr">fat pointer</span></td><td>A two word value carrying the address of some value, along with some further information necessary to put the value to use. Rust includes two kinds of <em>fat pointers</em>: references to slices, and trait objects. A reference to a slice carries the starting address of the slice and its length. A trait object carries a value's address and a pointer to the trait's implementation appropriate to that value. "Fat pointers" are also known as "wide pointers", and "double pointers".</td></tr> |
| <tr><td><span id="free-var">free variable</span></td><td>A <em>free variable</em> is one that is not bound within an expression or term; see <a href="appendix/./background.html#free-vs-bound">the background chapter for more</a></td></tr> |
| <tr><td><span id="generics">generics</span></td><td>The list of generic parameters defined on an item. There are three kinds of generic parameters: Type, lifetime and const parameters.</td></tr> |
| <tr><td><span id="hir">HIR</span></td><td>The <em>high-level <a href="appendix/glossary.html#ir">IR</a></em>, created by lowering and desugaring the AST. (<a href="appendix/../hir.html">see more</a>)</td></tr> |
| <tr><td><span id="hir-id"><code>HirId</code></span></td><td>Identifies a particular node in the HIR by combining a def-id with an "intra-definition offset". See <a href="appendix/../hir.html#identifiers-in-the-hir">the HIR chapter for more</a>.</td></tr> |
| <tr><td><span id="ice">ICE</span></td><td>Short for <em>internal compiler error</em>, this is when the compiler crashes.</td></tr> |
| <tr><td><span id="ich">ICH</span></td><td>Short for <em>incremental compilation hash</em>, these are used as fingerprints for things such as HIR and crate metadata, to check if changes have been made. This is useful in incremental compilation to see if part of a crate has changed and should be recompiled.</td></tr> |
| <tr><td><span id="infcx"><code>infcx</code></span></td><td>The type inference context (<code>InferCtxt</code>). (see <code>rustc_middle::infer</code>)</td></tr> |
| <tr><td><span id="inf-var">inference variable, infer var </span></td><td>When doing type, region, const inference, an <em>inference variable</em> is a kind of special type/region that represents what you are trying to infer. Think of X in algebra. For example, if we are trying to infer the type of a variable in a program, we create an inference variable to represent that unknown type.</td></tr> |
| <tr><td><span id="intern">intern</span></td><td>Interning refers to storing certain frequently-used constant data, such as strings, and then referring to the data by an identifier (e.g. a <code>Symbol</code>) rather than the data itself, to reduce memory usage and number of allocations. See <a href="appendix/../memory.html">this chapter</a> for more info.</td></tr> |
| <tr><td><span id="interpreter">interpreter</span></td><td>The heart of const evaluation, running MIR code at compile time. (<a href="appendix/../const-eval/interpret.html">see more</a>)</td></tr> |
| <tr><td><span id="intrinsic">intrinsic</span></td><td>Intrinsics are special functions that are implemented in the compiler itself but exposed (often unstably) to users. They do magical and dangerous things. (See <a href="https://doc.rust-lang.org/std/intrinsics/index.html"><code>std::intrinsics</code></a>)</td></tr> |
| <tr><td><span id="ir">IR</span></td><td>Short for <em>intermediate representation</em>, a general term in compilers. During compilation, the code is transformed from raw source (ASCII text) to various IRs. In Rust, these are primarily HIR, MIR, and LLVM IR. Each IR is well-suited for some set of computations. For example, MIR is well-suited for the borrow checker, and LLVM IR is well-suited for codegen because LLVM accepts it.</td></tr> |
| <tr><td><span id="irlo">IRLO, irlo</span></td><td>Sometimes used as an abbreviation for <a href="https://internals.rust-lang.org">internals.rust-lang.org</a>.</td></tr> |
| <tr><td><span id="item">item</span></td><td>A kind of "definition" in the language, such as a static, const, use statement, module, struct, etc. Concretely, this corresponds to the <code>Item</code> type.</td></tr> |
| <tr><td><span id="lang-item">lang item</span></td><td>Items that represent concepts intrinsic to the language itself, such as special built-in traits like <code>Sync</code> and <code>Send</code>; or traits representing operations such as <code>Add</code>; or functions that are called by the compiler. (<a href="https://doc.rust-lang.org/1.9.0/book/lang-items.html">see more</a>)</td></tr> |
| <tr><td><span id="lbl">late-bound lifetime</span></td><td>A lifetime region that is substituted at its call site. Bound in a HRTB and substituted by specific functions in the compiler, such as <code>liberate_late_bound_regions</code>. Contrast with <strong>early-bound lifetime</strong>. (<a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/region_kind/enum.RegionKind.html#bound-regions">see more</a>)</td></tr> |
| <tr><td><span id="local-crate">local crate</span></td><td>The crate currently being compiled. This is in contrast to "upstream crates" which refer to dependencies of the local crate.</td></tr> |
| <tr><td><span id="lto">LTO</span></td><td>Short for <em>link-time optimizations</em>, this is a set of optimizations offered by LLVM that occur just before the final binary is linked. These include optimizations like removing functions that are never used in the final program, for example. <em>ThinLTO</em> is a variant of LTO that aims to be a bit more scalable and efficient, but possibly sacrifices some optimizations. You may also read issues in the Rust repo about "FatLTO", which is the loving nickname given to non-Thin LTO. LLVM documentation: <a href="https://llvm.org/docs/LinkTimeOptimization.html">here</a> and <a href="https://clang.llvm.org/docs/ThinLTO.html">here</a>.</td></tr> |
| <tr><td><span id="llvm"><a href="https://llvm.org/">LLVM</a></span></td><td>(actually not an acronym :P) an open-source compiler backend. It accepts LLVM IR and outputs native binaries. Various languages (e.g. Rust) can then implement a compiler front-end that outputs LLVM IR and use LLVM to compile to all the platforms LLVM supports.</td></tr> |
| <tr><td><span id="memoization">memoization</span></td><td>The process of storing the results of (pure) computations (such as pure function calls) to avoid having to repeat them in the future. This is typically a trade-off between execution speed and memory usage.</td></tr> |
| <tr><td><span id="mir">MIR</span></td><td>The <em>mid-level <a href="appendix/glossary.html#ir">IR</a></em> that is created after type-checking for use by borrowck and codegen. (<a href="appendix/../mir/index.html">see more</a>)</td></tr> |
| <tr><td><span id="miri">Miri</span></td><td>A tool to detect Undefined Behavior in (unsafe) Rust code. (<a href="https://github.com/rust-lang/miri">see more</a>)</td></tr> |
| <tr><td><span id="mono">monomorphization</span></td><td>The process of taking generic implementations of types and functions and instantiating them with concrete types. For example, in the code we might have <code>Vec<T></code>, but in the final executable, we will have a copy of the <code>Vec</code> code for every concrete type used in the program (e.g. a copy for <code>Vec<usize></code>, a copy for <code>Vec<MyStruct></code>, etc).</td></tr> |
| <tr><td><span id="normalize">normalize</span></td><td>A general term for converting to a more canonical form, but in the case of rustc typically refers to <a href="appendix/../traits/goals-and-clauses.html#normalizeprojection---type">associated type normalization</a>.</td></tr> |
| <tr><td><span id="newtype">newtype</span></td><td>A wrapper around some other type (e.g., <code>struct Foo(T)</code> is a "newtype" for <code>T</code>). This is commonly used in Rust to give a stronger type for indices.</td></tr> |
| <tr><td><span id="niche">niche</span></td><td>Invalid bit patterns for a type <em>that can be used</em> for layout optimizations. Some types cannot have certain bit patterns. For example, the <code>NonZero*</code> integers or the reference <code>&T</code> cannot be represented by a 0 bitstring. This means the compiler can perform layout optimizations by taking advantage of the invalid "niche value". An example application for this is the <a href="https://rust-lang.github.io/unsafe-code-guidelines/layout/enums.html#discriminant-elision-on-option-like-enums"><em>Discriminant elision on <code>Option</code>-like enums</em></a>, which allows using a type's niche as the <a href="appendix/glossary.html#tag">"tag"</a> for an <code>enum</code> without requiring a separate field.</td></tr> |
| <tr><td><span id="nll">NLL</span></td><td>Short for <a href="appendix/../borrow_check/region_inference.html">non-lexical lifetimes</a>, this is an extension to Rust's borrowing system to make it be based on the control-flow graph.</td></tr> |
| <tr><td><span id="node-id">node-id or <code>NodeId</code></span></td><td>An index identifying a particular node in the AST or HIR; gradually being phased out and replaced with <code>HirId</code>. See <a href="appendix/../hir.html#identifiers-in-the-hir">the HIR chapter for more</a>.</td></tr> |
| <tr><td><span id="obligation">obligation</span></td><td>Something that must be proven by the trait system. (<a href="appendix/../traits/resolution.html">see more</a>)</td></tr> |
| <tr><td><span id="placeholder">placeholder</span></td><td><strong>NOTE: skolemization is deprecated by placeholder</strong> a way of handling subtyping around "for-all" types (e.g., <code>for<'a> fn(&'a u32)</code>) as well as solving higher-ranked trait bounds (e.g., <code>for<'a> T: Trait<'a></code>). See <a href="appendix/../borrow_check/region_inference/placeholders_and_universes.html">the chapter on placeholder and universes</a> for more details.</td></tr> |
| <tr><td><span id="point">point</span></td><td>Used in the NLL analysis to refer to some particular location in the MIR; typically used to refer to a node in the control-flow graph.</td></tr> |
| <tr><td><span id="projection">projection</span></td><td>A general term for a "relative path", e.g. <code>x.f</code> is a "field projection", and <code>T::Item</code> is an <a href="appendix/../traits/goals-and-clauses.html#trait-ref">"associated type projection"</a>.</td></tr> |
| <tr><td><span id="pc">promoted constants</span></td><td>Constants extracted from a function and lifted to static scope; see <a href="appendix/../mir/index.html#promoted">this section</a> for more details.</td></tr> |
| <tr><td><span id="provider">provider</span></td><td>The function that executes a query. (<a href="appendix/../query.html">see more</a>)</td></tr> |
| <tr><td><span id="quantified">quantified</span></td><td>In math or logic, existential and universal quantification are used to ask questions like "is there any type T for which is true?" or "is this true for all types T?"; see <a href="appendix/./background.html#quantified">the background chapter for more</a>.</td></tr> |
| <tr><td><span id="query">query</span></td><td>A sub-computation during compilation. Query results can be cached in the current session or to disk for incremental compilation. (<a href="appendix/../query.html">see more</a>)</td></tr> |
| <tr><td><span id="recovery">recovery</span></td><td>Recovery refers to handling invalid syntax during parsing (e.g. a missing comma) and continuing to parse the AST. This avoid showing spurious errors to the user (e.g. showing 'missing field' errors when the struct definition contains errors).</td></tr> |
| <tr><td><span id="region">region</span></td><td>Another term for "lifetime" often used in the literature and in the borrow checker.</td></tr> |
| <tr><td><span id="rib">rib</span></td><td>A data structure in the name resolver that keeps track of a single scope for names. (<a href="appendix/../name-resolution.html">see more</a>)</td></tr> |
| <tr><td><span id="rpit">RPIT</span></td><td>A return-position <code>impl Trait</code>. (<a href="https://doc.rust-lang.org/reference/types/impl-trait.html#abstract-return-types">see the reference</a>).</td></tr> |
| <tr><td><span id="rpitit">RPITIT</span></td><td>A return-position <code>impl Trait</code> in trait. Unlike RPIT, this is desugared to a generic associated type (GAT). Introduced in <a href="https://rust-lang.github.io/rfcs/3425-return-position-impl-trait-in-traits.html">RFC 3425</a>. (<a href="appendix/../return-position-impl-trait-in-trait.html">see more</a>)</td></tr> |
| <tr><td><span id="rustbuild">rustbuild</span></td><td>A deprecated term for the part of bootstrap that is written in Rust</td></tr> |
| <tr><td><span id="scrutinee">scrutinee</span></td><td>A scrutinee is the expression that is matched on in <code>match</code> expressions and similar pattern matching constructs. For example, in <code>match x { A => 1, B => 2 }</code>, the expression <code>x</code> is the scrutinee.</td></tr> |
| <tr><td><span id="sess"><code>sess</code></span></td><td>The compiler <em>session</em>, which stores global data used throughout compilation</td></tr> |
| <tr><td><span id="side-tables">side tables</span></td><td>Because the <a href="appendix/glossary.html#ast">AST</a> and HIR are immutable once created, we often carry extra information about them in the form of hashtables, indexed by the id of a particular node.</td></tr> |
| <tr><td><span id="sigil">sigil</span></td><td>Like a keyword but composed entirely of non-alphanumeric tokens. For example, <code>&</code> is a sigil for references.</td></tr> |
| <tr><td><span id="soundness">soundness</span></td><td>A technical term in type theory. Roughly, if a type system is sound, then a program that type-checks is type-safe. That is, one can never (in safe rust) force a value into a variable of the wrong type. (see "completeness").</td></tr> |
| <tr><td><span id="span">span</span></td><td>A location in the user's source code, used for error reporting primarily. These are like a file-name/line-number/column tuple on steroids: they carry a start/end point, and also track macro expansions and compiler desugaring. All while being packed into a few bytes (really, it's an index into a table). See the <a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html"><code>Span</code></a> datatype for more.</td></tr> |
| <tr><td><span id="subst">subst</span></td><td>The act of <em>substituting</em> the generic parameters inside of a type, constant expression, etc. with concrete generic arguments by supplying <a href="appendix/glossary.html#substs">substs</a>. Nowadays referred to as <em>instantiating</em> in the compiler.</td></tr> |
| <tr><td><span id="substs">substs</span></td><td>The <em>substitutions</em> for a given generic item (e.g. the <code>i32</code>, <code>u32</code> in <code>HashMap<i32, u32></code>). Nowadays referred to as the list of <em>generic arguments</em> in the compiler (but note that strictly speaking these two concepts differ, see the literature).</td></tr> |
| <tr><td><span id="sysroot">sysroot</span></td><td>The directory for build artifacts that are loaded by the compiler at runtime. (<a href="appendix/../building/bootstrapping/what-bootstrapping-does.html#what-is-a-sysroot">see more</a>)</td></tr> |
| <tr><td><span id="tag">tag</span></td><td>The "tag" of an enum/generator encodes the <a href="appendix/glossary.html#discriminant">discriminant</a> of the active variant/state. Tags can either be "direct" (simply storing the discriminant in a field) or use a <a href="appendix/glossary.html#niche">"niche"</a>.</td></tr> |
| <tr><td><span id="tait">TAIT</span></td><td>A type-alias <code>impl Trait</code>. Introduced in <a href="https://rust-lang.github.io/rfcs/2515-type_alias_impl_trait.html">RFC 2515</a>.</td></tr> |
| <tr><td><span id="tcx"><code>tcx</code></span></td><td>Standard variable name for the "typing context" (<code>TyCtxt</code>), main data structure of the compiler. (<a href="appendix/../ty.html">see more</a>)</td></tr> |
| <tr><td><span id="lifetime-tcx"><code>'tcx</code></span></td><td>The lifetime of the allocation arenas used by <code>TyCtxt</code>. Most data interned during a compilation session will use this lifetime with the exception of HIR data which uses the <code>'hir</code> lifetime. (<a href="appendix/../ty.html">see more</a>)</td></tr> |
| <tr><td><span id="token">token</span></td><td>The smallest unit of parsing. Tokens are produced after lexing (<a href="appendix/../the-parser.html">see more</a>).</td></tr> |
| <tr><td><span id="tls"><a href="https://llvm.org/docs/LangRef.html#thread-local-storage-models">TLS</a></span></td><td><em>Thread-local storage</em>. Variables may be defined so that each thread has its own copy (rather than all threads sharing the variable). This has some interactions with LLVM. Not all platforms support TLS.</td></tr> |
| <tr><td><span id="trait-ref">trait reference, trait ref </span></td><td>The name of a trait along with a suitable list of generic arguments. (<a href="appendix/../traits/goals-and-clauses.html#trait-ref">see more</a>)</td></tr> |
| <tr><td><span id="trans">trans</span></td><td>Short for <em>translation</em>, the code to translate MIR into LLVM IR. Renamed to <a href="appendix/glossary.html#codegen">codegen</a>.</td></tr> |
| <tr><td><span id="ty"><code>Ty</code></span></td><td>The internal representation of a type. (<a href="appendix/../ty.html">see more</a>)</td></tr> |
| <tr><td><span id="tyctxt"><code>TyCtxt</code></span></td><td>The data structure often referred to as <a href="appendix/glossary.html#tcx"><code>tcx</code></a> in code which provides access to session data and the query system.</td></tr> |
| <tr><td><span id="ufcs">UFCS</span></td><td>Short for <em>universal function call syntax</em>, this is an unambiguous syntax for calling a method. <strong>Term no longer in use!</strong> Prefer <em>fully-qualified path/syntax</em>. (<a href="appendix/../type-checking.html">see more</a>, <a href="https://doc.rust-lang.org/reference/expressions/call-expr.html#disambiguating-function-calls">see the reference</a>)</td></tr> |
| <tr><td><span id="ut">uninhabited type</span></td><td>A type which has <em>no</em> values. This is not the same as a ZST, which has exactly 1 value. An example of an uninhabited type is <code>enum Foo {}</code>, which has no variants, and so, can never be created. The compiler can treat code that deals with uninhabited types as dead code, since there is no such value to be manipulated. <code>!</code> (the never type) is an uninhabited type. Uninhabited types are also called <em>empty types</em>.</td></tr> |
| <tr><td><span id="upvar">upvar</span></td><td>A variable captured by a closure from outside the closure.</td></tr> |
| <tr><td><span id="variance">variance</span></td><td>Determines how changes to a generic parameter affect subtyping; for example, if <code>T</code> is a subtype of <code>U</code>, then <code>Vec<T></code> is a subtype <code>Vec<U></code> because <code>Vec</code> is <em>covariant</em> in its generic parameter. See <a href="appendix/./background.html#variance">the background chapter</a> for a more general explanation. See the <a href="appendix/../variance.html">variance chapter</a> for an explanation of how type checking handles variance.</td></tr> |
| <tr><td><span id="variant-idx">variant index</span></td><td>In an enum, identifies a variant by assigning them indices starting at 0. This is purely internal and not to be confused with the <a href="appendix/glossary.html#discriminant">"discriminant"</a> which can be overwritten by the user (e.g. <code>enum Bool { True = 42, False = 0 }</code>).</td></tr> |
| <tr><td><span id="wf">well-formedness</span></td><td>Semantically: An expression that evaluates to meaningful result. In type systems: A type related construct which follows rules of the type system.</td></tr> |
| <tr><td><span id="wide-ptr">wide pointer</span></td><td>A pointer with additional metadata. See <a href="appendix/glossary.html#fat-ptr">fat pointer</a> for more.</td></tr> |
| <tr><td><span id="zst">ZST</span></td><td><em>Zero-sized type</em>. A type whose values have size 0 bytes. Since <code>2^0 = 1</code>, such types can have exactly one value. For example, <code>()</code> (unit) is a ZST. <code>struct Foo;</code> is also a ZST. The compiler can do some nice optimizations around ZSTs.</td></tr> |
| </tbody></table> |
| </div><div style="break-before: page; page-break-before: always;"></div><h1 id="code-index"><a class="header" href="#code-index">Code Index</a></h1> |
| <p>rustc has a lot of important data structures. This is an attempt to give some |
| guidance on where to learn more about some of the key data structures of the |
| compiler.</p> |
| <div class="table-wrapper"><table><thead><tr><th>Item</th><th>Kind</th><th>Short description</th><th>Chapter</th><th>Declaration</th></tr></thead><tbody> |
| <tr><td><code>BodyId</code></td><td>struct</td><td>One of four types of HIR node identifiers</td><td><a href="appendix/../hir.html#hir-id">Identifiers in the HIR</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.BodyId.html">compiler/rustc_hir/src/hir.rs</a></td></tr> |
| <tr><td><code>Compiler</code></td><td>struct</td><td>Represents a compiler session and can be used to drive a compilation.</td><td><a href="appendix/../rustc-driver/intro.html">The Rustc Driver and Interface</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Compiler.html">compiler/rustc_interface/src/interface.rs</a></td></tr> |
| <tr><td><code>ast::Crate</code></td><td>struct</td><td>A syntax-level representation of a parsed crate</td><td><a href="appendix/../the-parser.html">The parser</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ast/struct.Crate.html">compiler/rustc_ast/src/ast.rs</a></td></tr> |
| <tr><td><code>rustc_hir::Crate</code></td><td>struct</td><td>A more abstract, compiler-friendly form of a crate's AST</td><td><a href="appendix/../hir.html">The Hir</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Crate.html">compiler/rustc_hir/src/hir.rs</a></td></tr> |
| <tr><td><code>DefId</code></td><td>struct</td><td>One of four types of HIR node identifiers</td><td><a href="appendix/../hir.html#hir-id">Identifiers in the HIR</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html">compiler/rustc_hir/src/def_id.rs</a></td></tr> |
| <tr><td><code>Diag</code></td><td>struct</td><td>A struct for a compiler diagnostic, such as an error or lint</td><td><a href="appendix/../diagnostics.html">Emitting Diagnostics</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html">compiler/rustc_errors/src/diagnostic.rs</a></td></tr> |
| <tr><td><code>DocContext</code></td><td>struct</td><td>A state container used by rustdoc when crawling through a crate to gather its documentation</td><td><a href="appendix/../rustdoc.html">Rustdoc</a></td><td><a href="https://github.com/rust-lang/rust/blob/HEAD/src/librustdoc/core.rs">src/librustdoc/core.rs</a></td></tr> |
| <tr><td><code>HirId</code></td><td>struct</td><td>One of four types of HIR node identifiers</td><td><a href="appendix/../hir.html#hir-id">Identifiers in the HIR</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.HirId.html">compiler/rustc_hir_id/src/lib.rs</a></td></tr> |
| <tr><td><code>Lexer</code></td><td>struct</td><td>This is the lexer used during parsing. It consumes characters from the raw source code being compiled and produces a series of tokens for use by the rest of the parser</td><td><a href="appendix/../the-parser.html">The parser</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html">compiler/rustc_parse/src/lexer/mod.rs</a></td></tr> |
| <tr><td><code>NodeId</code></td><td>struct</td><td>One of four types of HIR node identifiers. Being phased out</td><td><a href="appendix/../hir.html#hir-id">Identifiers in the HIR</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html">compiler/rustc_ast/src/ast.rs</a></td></tr> |
| <tr><td><code>P</code></td><td>struct</td><td>An owned immutable smart pointer. By contrast, <code>&T</code> is not owned, and <code>Box<T></code> is not immutable.</td><td>None</td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ptr/struct.P.html">compiler/rustc_ast/src/ptr.rs</a></td></tr> |
| <tr><td><code>ParamEnv</code></td><td>struct</td><td>Information about generic parameters or <code>Self</code>, useful for working with associated or generic items</td><td><a href="appendix/../typing_parameter_envs.html">Parameter Environment</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html">compiler/rustc_middle/src/ty/mod.rs</a></td></tr> |
| <tr><td><code>ParseSess</code></td><td>struct</td><td>This struct contains information about a parsing session</td><td><a href="appendix/../the-parser.html">The parser</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html">compiler/rustc_session/src/parse/parse.rs</a></td></tr> |
| <tr><td><code>Rib</code></td><td>struct</td><td>Represents a single scope of names</td><td><a href="appendix/../name-resolution.html">Name resolution</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html">compiler/rustc_resolve/src/lib.rs</a></td></tr> |
| <tr><td><code>Session</code></td><td>struct</td><td>The data associated with a compilation session</td><td><a href="appendix/../the-parser.html">The parser</a>, <a href="appendix/../rustc-driver/intro.html">The Rustc Driver and Interface</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html">compiler/rustc_session/src/session.rs</a></td></tr> |
| <tr><td><code>SourceFile</code></td><td>struct</td><td>Part of the <code>SourceMap</code>. Maps AST nodes to their source code for a single source file. Was previously called FileMap</td><td><a href="appendix/../the-parser.html">The parser</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.SourceFile.html">compiler/rustc_span/src/lib.rs</a></td></tr> |
| <tr><td><code>SourceMap</code></td><td>struct</td><td>Maps AST nodes to their source code. It is composed of <code>SourceFile</code>s. Was previously called CodeMap</td><td><a href="appendix/../the-parser.html">The parser</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html">compiler/rustc_span/src/source_map.rs</a></td></tr> |
| <tr><td><code>Span</code></td><td>struct</td><td>A location in the user's source code, used for error reporting primarily</td><td><a href="appendix/../diagnostics.html">Emitting Diagnostics</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html">compiler/rustc_span/src/span_encoding.rs</a></td></tr> |
| <tr><td><code>rustc_ast::token_stream::TokenStream</code></td><td>struct</td><td>An abstract sequence of tokens, organized into <code>TokenTree</code>s</td><td><a href="appendix/../the-parser.html">The parser</a>, <a href="appendix/../macro-expansion.html">Macro expansion</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html">compiler/rustc_ast/src/tokenstream.rs</a></td></tr> |
| <tr><td><code>TraitDef</code></td><td>struct</td><td>This struct contains a trait's definition with type information</td><td><a href="appendix/../ty.html">The <code>ty</code> modules</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait_def/struct.TraitDef.html">compiler/rustc_middle/src/ty/trait_def.rs</a></td></tr> |
| <tr><td><code>TraitRef</code></td><td>struct</td><td>The combination of a trait and its input types (e.g. <code>P0: Trait<P1...Pn></code>)</td><td><a href="appendix/../traits/goals-and-clauses.html#domain-goals">Trait Solving: Goals and Clauses</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TraitRef.html">compiler/rustc_middle/src/ty/sty.rs</a></td></tr> |
| <tr><td><code>Ty<'tcx></code></td><td>struct</td><td>This is the internal representation of a type used for type checking</td><td><a href="appendix/../type-checking.html">Type checking</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html">compiler/rustc_middle/src/ty/mod.rs</a></td></tr> |
| <tr><td><code>TyCtxt<'tcx></code></td><td>struct</td><td>The "typing context". This is the central data structure in the compiler. It is the context that you use to perform all manner of queries</td><td><a href="appendix/../ty.html">The <code>ty</code> modules</a></td><td><a href="https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html">compiler/rustc_middle/src/ty/context.rs</a></td></tr> |
| </tbody></table> |
| </div><div style="break-before: page; page-break-before: always;"></div><h1 id="compiler-lecture-series"><a class="header" href="#compiler-lecture-series">Compiler Lecture Series</a></h1> |
| <p>These are videos where various experts explain different parts of the compiler:</p> |
| <h2 id="general"><a class="header" href="#general">General</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=elBxMRSNYr4">January 2019: Tom Tromey discusses debugging support in rustc</a></li> |
| <li><a href="https://www.youtube.com/watch?v=N6b44kMS6OM">June 2019: Responsive compilers - Nicholas Matsakis - PLISS 2019</a></li> |
| <li><a href="https://www.youtube.com/watch?v=LIYkT3p5gTs">June 2019: Things I Learned (TIL) - Nicholas Matsakis - PLISS 2019</a></li> |
| </ul> |
| <h2 id="rust-analyzer"><a class="header" href="#rust-analyzer">Rust Analyzer</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=_muY4HjSqVw">January 2019: How Salsa Works</a></li> |
| <li><a href="https://www.youtube.com/watch?v=i_IhACacPRY">January 2019: Salsa In More Depth</a></li> |
| <li><a href="https://www.youtube.com/watch?v=ANKBNiSWyfc">January 2019: Rust analyzer guide</a></li> |
| <li><a href="https://www.youtube.com/watch?v=DGAuLWdCCAI">February 2019: Rust analyzer syntax trees</a></li> |
| <li><a href="https://www.youtube.com/watch?v=Lmp3P9WNL8o">March 2019: rust-analyzer type-checker overview by flodiebold</a></li> |
| <li><a href="https://www.youtube.com/watch?v=Xr-rBqLr-G4">March 2019: RLS 2.0, Salsa, and Name Resolution</a></li> |
| </ul> |
| <h2 id="type-system"><a class="header" href="#type-system">Type System</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=Q7lQCgnNWU0">July 2015: Felix Klock - Rust: A type system you didn't know you wanted - Curry On</a></li> |
| <li><a href="https://www.youtube.com/watch?v=fI4RG_uq-WU">November 2016: Felix Klock - Subtyping in Rust and Clarke's Third Law</a></li> |
| <li><a href="https://www.youtube.com/watch?v=iV1Z0xYXkck">February 2019: Universes and Lifetimes</a></li> |
| <li><a href="https://www.youtube.com/watch?v=c01TsOsr3-c">April 2019: Representing types in rustc</a></li> |
| <li><a href="https://www.youtube.com/watch?v=UTXOptVMuIc">March 2019: RFC #2229 Disjoint Field Capture plan</a></li> |
| </ul> |
| <h2 id="closures"><a class="header" href="#closures">Closures</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=fMopdkn5-Xw">October 2018: closures and upvar capture</a></li> |
| <li><a href="https://www.youtube.com/watch?v=pLmVhSB-z4s">October 2018: blitzerr closure upvar tys</a></li> |
| <li><a href="https://www.youtube.com/watch?v=2QCuNtISoYc">January 2019: Convert Closure Upvar Representation to Tuples with blitzerr</a></li> |
| </ul> |
| <h2 id="chalk"><a class="header" href="#chalk">Chalk</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=rZqS4bLPL24">July 2018: Coherence in Chalk by Sunjay Varma - Bay Area Rust Meetup</a></li> |
| <li><a href="https://www.youtube.com/watch?v=MBWtbDifPeU">March 2019: rustc-chalk integration overview</a></li> |
| <li><a href="https://www.youtube.com/watch?v=Ny2928cGDoM">April 2019: How the chalk-engine crate works</a></li> |
| <li><a href="https://www.youtube.com/watch?v=hmV66tB79LM">May 2019: How the chalk-engine crate works 2</a></li> |
| </ul> |
| <h2 id="polonius"><a class="header" href="#polonius">Polonius</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=i5KdU0ieb_A">March 2019: Polonius-rustc walkthrough</a></li> |
| <li><a href="https://www.youtube.com/watch?v=ilv9V-328HI">May 2019: Polonius WG: Initialization and move tracking</a></li> |
| </ul> |
| <h2 id="miri"><a class="header" href="#miri">Miri</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=5Pm2C1YXrvM">March 2019: oli-obk on miri and constant evaluation</a></li> |
| </ul> |
| <h2 id="async"><a class="header" href="#async">Async</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=xe2_whJWBC0">February 2019: async-await implementation plans</a></li> |
| <li><a href="https://www.youtube.com/watch?v=hlOxfkUDLPQ">April 2019: async-await region inferencer</a></li> |
| </ul> |
| <h2 id="code-generation-3"><a class="header" href="#code-generation-3">Code Generation</a></h2> |
| <ul> |
| <li><a href="https://www.youtube.com/watch?v=9OIA7DTFQWU">January 2019: Cranelift</a></li> |
| <li><a href="https://www.youtube.com/watch?v=Kqz-umsAnk8">December 2024: LLVM Developers' Meeting - Rust ❤️ LLVM</a></li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="rust-bibliography"><a class="header" href="#rust-bibliography">Rust Bibliography</a></h1> |
| <p>This is a reading list of material relevant to Rust. It includes prior |
| research that has - at one time or another - influenced the design of |
| Rust, as well as publications about Rust.</p> |
| <h2 id="type-system-1"><a class="header" href="#type-system-1">Type system</a></h2> |
| <ul> |
| <li><a href="https://dl.acm.org/doi/10.1002/spe.370">Alias burying</a> - We tried something similar and abandoned it.</li> |
| <li><a href="https://lirias.kuleuven.be/retrieve/35835">External uniqueness is unique enough</a></li> |
| <li><a href="https://www.cs.utah.edu/plt/publications/jfp12-draft-fcdf.pdf">Macros that work together</a></li> |
| <li><a href="https://dl.acm.org/doi/10.1145/75277.75283">Making ad-hoc polymorphism less ad hoc</a></li> |
| <li><a href="https://www.cs.umd.edu/projects/cyclone/papers/cyclone-regions.pdf">Region based memory management in Cyclone</a></li> |
| <li><a href="https://www.cs.ucla.edu/~palsberg/tba/papers/tofte-talpin-iandc97.pdf">Region Based Memory Management</a></li> |
| <li><a href="https://www.cs.umd.edu/projects/PL/cyclone/scp.pdf">Safe manual memory management in Cyclone</a></li> |
| <li><a href="https://en.wikipedia.org/wiki/Skolem_normal_form">Skolem Normal Form</a></li> |
| <li><a href="http://scg.unibe.ch/archive/papers/Scha03aTraits.pdf">Traits: composable units of behavior</a></li> |
| <li><a href="https://research.microsoft.com/pubs/170528/msr-tr-2012-79.pdf">Uniqueness and Reference Immutability for Safe Parallelism</a></li> |
| </ul> |
| <h2 id="concurrency"><a class="header" href="#concurrency">Concurrency</a></h2> |
| <ul> |
| <li><a href="https://web.archive.org/web/20190904045322/http://www.coopsoft.com/ar/CalamityArticle.html">A Java fork/join calamity</a> - critique of Java's fork/join library, particularly its application of work stealing to non-strict computation</li> |
| <li><a href="https://www.cs.rochester.edu/u/scott/papers/1991_TOCS_synch.pdf">Algorithms for scalable synchronization of shared-memory multiprocessors</a></li> |
| <li><a href="https://web.njit.edu/~dingxn/papers/BWS.pdf">Balanced work stealing for time-sharing multicores</a></li> |
| <li><a href="https://www.blagodurov.net/files/a8-blagodurov.pdf">Contention aware scheduling</a></li> |
| <li><a href="https://patents.google.com/patent/US7346753B2/en">Dynamic circular work stealing deque</a> - The Chase/Lev deque</li> |
| <li><a href="https://www.cl.cam.ac.uk/techreports/UCAM-CL-TR-579.pdf">Epoch-based reclamation</a>.</li> |
| <li><a href="https://research.microsoft.com/pubs/67482/singsharp.pdf">Language support for fast and reliable message passing in singularity OS</a></li> |
| <li><a href="https://www.cs.bgu.ac.il/%7Ehendlerd/papers/p280-hendler.pdf">Non-blocking steal-half work queues</a></li> |
| <li><a href="https://aturon.github.io/academic/reagents.pdf">Reagents: expressing and composing fine-grained concurrency</a></li> |
| <li><a href="https://www.lri.fr/~cecile/ENSEIGNEMENT/IPAR/Exposes/cilk1.pdf">Scheduling multithreaded computations by work stealing</a></li> |
| <li><a href="https://www.stanford.edu/~ouster/cgi-bin/papers/coscheduling.pdf">Scheduling techniques for concurrent systems</a></li> |
| <li><a href="https://research.microsoft.com/pubs/69431/osr2007_rethinkingsoftwarestack.pdf">Singularity: rethinking the software stack</a></li> |
| <li><a href="http://www.aladdin.cs.cmu.edu/papers/pdfs/y2000/locality_spaa00.pdf">The data locality of work stealing</a></li> |
| <li><a href="https://www.eecis.udel.edu/%7Ecavazos/cisc879-spring2008/papers/arora98thread.pdf">Thread scheduling for multiprogramming multiprocessors</a></li> |
| <li><a href="https://dl.acm.org/doi/10.1145/1953611.1953616">Three layer cake for shared-memory programming</a></li> |
| <li><a href="https://dl.acm.org/doi/10.1109/IPDPS.2009.5161079">Work-first and help-first scheduling policies for async-finish task parallelism</a> - More general than fully-strict work stealing</li> |
| </ul> |
| <h2 id="others"><a class="header" href="#others">Others</a></h2> |
| <ul> |
| <li><a href="https://people.cs.umass.edu/~emery/pubs/berger-pldi2001.pdf">Composing High-Performance Memory Allocators</a></li> |
| <li><a href="https://www.usenix.org/legacy/events/hotos03/tech/full_papers/candea/candea.pdf">Crash-only software</a></li> |
| <li><a href="https://people.cs.umass.edu/~emery/pubs/berger-oopsla2002.pdf">Reconsidering Custom Memory Allocation</a></li> |
| </ul> |
| <h2 id="papers-about-rust"><a class="header" href="#papers-about-rust">Papers <em>about</em> Rust</a></h2> |
| <ul> |
| <li><a href="https://ieeexplore.ieee.org/document/6650903">GPU Programming in Rust: Implementing High Level Abstractions in a Systems |
| Level |
| Language</a>. |
| Early GPU work by Eric Holk.</li> |
| <li><a href="https://www.usenix.org/conference/hotpar12/parallel-closures-new-twist-old-idea">Parallel closures: a new twist on an old |
| idea</a> |
| <ul> |
| <li>not exactly about Rust, but by nmatsakis</li> |
| </ul> |
| </li> |
| <li><a href="https://dada.cs.washington.edu/research/tr/2015/03/UW-CSE-15-03-02.pdf">Patina: A Formalization of the Rust Programming |
| Language</a>. |
| Early formalization of a subset of the type system, by Eric Reed.</li> |
| <li><a href="https://arxiv.org/abs/1505.07383">Experience Report: Developing the Servo Web Browser Engine using |
| Rust</a>. By Lars Bergstrom.</li> |
| <li><a href="https://michaelsproul.github.io/rust_radix_paper/rust-radix-sproul.pdf">Implementing a Generic Radix Trie in |
| Rust</a>. Undergrad |
| paper by Michael Sproul.</li> |
| <li><a href="https://scialex.github.io/reenix.pdf">Reenix: Implementing a Unix-Like Operating System in |
| Rust</a>. Undergrad paper by Alex |
| Light.</li> |
| <li><a href="https://github.com/1wilkens/thesis-ba">Evaluation of performance and productivity metrics of potential programming languages in the HPC environment</a>. |
| Bachelor's thesis by Florian Wilkens. Compares C, Go and Rust.</li> |
| <li><a href="http://spw15.langsec.org/papers/couprie-nom.pdf">Nom, a byte oriented, streaming, zero copy, parser combinators library |
| in Rust</a>. By |
| Geoffroy Couprie, research for VLC.</li> |
| <li><a href="https://compilers.cs.uni-saarland.de/papers/lkh15_cgo.pdf">Graph-Based Higher-Order Intermediate |
| Representation</a>. An |
| experimental IR implemented in Impala, a Rust-like language.</li> |
| <li><a href="https://compilers.cs.uni-saarland.de/papers/ppl14_web.pdf">Code Refinement of Stencil |
| Codes</a>. Another |
| paper using Impala.</li> |
| <li><a href="http://publications.lib.chalmers.se/records/fulltext/219016/219016.pdf">Parallelization in Rust with fork-join and |
| friends</a>. Linus |
| Farnstrand's master's thesis.</li> |
| <li><a href="https://munksgaard.me/papers/laumann-munksgaard-larsen.pdf">Session Types for |
| Rust</a>. Philip |
| Munksgaard's master's thesis. Research for Servo.</li> |
| <li><a href="https://amitlevy.com/papers/tock-plos2015.pdf">Ownership is Theft: Experiences Building an Embedded OS in Rust - Amit Levy, et. al.</a></li> |
| <li><a href="https://faultlore.com/blah/papers/thesis.pdf">You can't spell trust without Rust</a>. Aria Beingessner's master's thesis.</li> |
| <li><a href="https://rust-bio.github.io/">Rust-Bio: a fast and safe bioinformatics library</a>. Johannes Köster</li> |
| <li><a href="https://csperkins.org/research/thesis-msci-clipsham.pdf">Safe, Correct, and Fast Low-Level Networking</a>. Robert Clipsham's master's thesis.</li> |
| <li><a href="https://open.library.ubc.ca/cIRcle/collections/ubctheses/24/items/1.0220521">Formalizing Rust traits</a>. Jonatan Milewski's master's thesis.</li> |
| <li><a href="https://dl.acm.org/doi/pdf/10.1145/3241624.2926707">Rust as a Language for High Performance GC Implementation</a></li> |
| <li><a href="https://github.com/Kha/electrolysis">Simple Verification of Rust Programs via Functional Purification</a>. Sebastian Ullrich's master's thesis.</li> |
| <li><a href="http://spw17.langsec.org/papers/chifflier-parsing-in-2017.pdf">Writing parsers like it is 2017</a> Pierre Chifflier and Geoffroy Couprie for the Langsec Workshop</li> |
| <li><a href="https://www.tockos.org/assets/papers/rust-kernel-apsys2017.pdf">The Case for Writing a Kernel in Rust</a></li> |
| <li><a href="https://plv.mpi-sws.org/rustbelt/popl18/">RustBelt: Securing the Foundations of the Rust Programming Language</a></li> |
| <li><a href="https://arxiv.org/abs/1903.00982">Oxide: The Essence of Rust</a>. By Aaron Weiss, Olek Gierczak, Daniel Patterson, Nicholas D. Matsakis, and Amal Ahmed.</li> |
| </ul> |
| <div style="break-before: page; page-break-before: always;"></div><h1 id="humor-in-rust"><a class="header" href="#humor-in-rust">Humor in Rust</a></h1> |
| <p>What's a project without a sense of humor? And frankly some of these are |
| enlightening?</p> |
| <ul> |
| <li><a href="https://github.com/rust-lang/rust/blob/HEAD/tests/ui/expr/weird-exprs.rs">Weird exprs test</a></li> |
| <li><a href="https://fitzgen.com/2018/12/13/rust-raps.html">Ferris Rap</a></li> |
| <li><a href="https://github.com/rust-lang/rust/pull/53645#issue-210543221">The Genesis of Generic Germination</a></li> |
| <li><a href="https://github.com/rust-lang/rust/blob/79d8a0fcefa5134db2a94739b1d18daa01fc6e9f/src/test/ui/bastion-of-the-turbofish.rs">The Bastion of the Turbofish test</a></li> |
| <li><a href="https://users.rust-lang.org/t/rust-koans/2408">Rust Koans</a></li> |
| <li><a href="https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=0ab2bd6a9d722e0f05a95e2a5dcf89cc"><code>break rust;</code></a></li> |
| <li><a href="https://doc.rust-lang.org/stable/nomicon/">The Nomicon Intro</a></li> |
| <li><a href="https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/rustc-ty.20naming.20bikeshed.20.2F.20punfest.20%28was.3A.20design.20meeting.202.2E.2E.2E/near/189906455"><code>rustc-ty</code> renaming punfest</a></li> |
| <li><a href="https://github.com/rust-lang/rust/pull/91476">try using their name "ferris" instead</a></li> |
| <li><a href="https://github.com/rust-lang/rust/pull/70645">Forbid pineapple on pizza</a></li> |
| </ul> |
| |
| </main> |
| |
| <nav class="nav-wrapper" aria-label="Page navigation"> |
| <!-- Mobile navigation buttons --> |
| |
| |
| <div style="clear: both"></div> |
| </nav> |
| </div> |
| </div> |
| |
| <nav class="nav-wide-wrapper" aria-label="Page navigation"> |
| |
| </nav> |
| |
| </div> |
| |
| |
| |
| |
| <script> |
| window.playground_copyable = true; |
| </script> |
| |
| |
| <script src="elasticlunr.min.js"></script> |
| <script src="mark.min.js"></script> |
| <script src="searcher.js"></script> |
| |
| <script src="clipboard.min.js"></script> |
| <script src="highlight.js"></script> |
| <script src="book.js"></script> |
| |
| <!-- Custom JS scripts --> |
| <script src="mermaid.min.js"></script> |
| <script src="mermaid-init.js"></script> |
| <script src="pagetoc.js"></script> |
| |
| <script> |
| window.addEventListener('load', function() { |
| window.setTimeout(window.print, 100); |
| }); |
| </script> |
| |
| |
| </div> |
| </body> |
| </html> |
