tests/adding.html - rust-lang/rustc-dev-guide - Git at Google

 <!DOCTYPE HTML>
 <html lang="en" class="light sidebar-visible" dir="ltr">
     <head>
         <!-- Book generated using mdBook -->
         <meta charset="UTF-8">
         <title>Adding new tests - Rust Compiler Development Guide</title>


         <!-- Custom HTML head -->

         <meta name="description" content="A guide to developing the Rust compiler (rustc)">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <meta name="theme-color" content="#ffffff">

         <link rel="icon" href="../favicon.svg">
         <link rel="shortcut icon" href="../favicon.png">
         <link rel="stylesheet" href="../css/variables.css">
         <link rel="stylesheet" href="../css/general.css">
         <link rel="stylesheet" href="../css/chrome.css">
         <link rel="stylesheet" href="../css/print.css" media="print">

         <!-- Fonts -->
         <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
         <link rel="stylesheet" href="../fonts/fonts.css">

         <!-- Highlight.js Stylesheets -->
         <link rel="stylesheet" id="highlight-css" href="../highlight.css">
         <link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
         <link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">

         <!-- Custom theme stylesheets -->


         <!-- Provide site root and default themes to javascript -->
         <script>
             const path_to_root = "../";
             const default_light_theme = "light";
             const default_dark_theme = "navy";
         </script>
         <!-- Start loading toc.js asap -->
         <script src="../toc.js"></script>
     </head>
     <body>
     <div id="body-container">
         <!-- Work around some values being stored in localStorage wrapped in quotes -->
         <script>
             try {
                 let theme = localStorage.getItem('mdbook-theme');
                 let sidebar = localStorage.getItem('mdbook-sidebar');

                 if (theme.startsWith('"') && theme.endsWith('"')) {
                     localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
                 }

                 if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
                     localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
                 }
             } catch (e) { }
         </script>

         <!-- Set the theme before any content is loaded, prevents flash -->
         <script>
             const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
             let theme;
             try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
             if (theme === null || theme === undefined) { theme = default_theme; }
             const html = document.documentElement;
             html.classList.remove('light')
             html.classList.add(theme);
             html.classList.add("js");
         </script>

         <input type="checkbox" id="sidebar-toggle-anchor" class="hidden">

         <!-- Hide / unhide sidebar before it is displayed -->
         <script>
             let sidebar = null;
             const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
             if (document.body.clientWidth >= 1080) {
                 try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
                 sidebar = sidebar || 'visible';
             } else {
                 sidebar = 'hidden';
             }
             sidebar_toggle.checked = sidebar === 'visible';
             html.classList.remove('sidebar-visible');
             html.classList.add("sidebar-" + sidebar);
         </script>

         <nav id="sidebar" class="sidebar" aria-label="Table of contents">
             <!-- populated by js -->
             <mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
             <noscript>
                 <iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
             </noscript>
             <div id="sidebar-resize-handle" class="sidebar-resize-handle">
                 <div class="sidebar-resize-indicator"></div>
             </div>
         </nav>

         <div id="page-wrapper" class="page-wrapper">

             <div class="page">
                 <div id="menu-bar-hover-placeholder"></div>
                 <div id="menu-bar" class="menu-bar sticky">
                     <div class="left-buttons">
                         <label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
                             <i class="fa fa-bars"></i>
                         </label>
                         <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
                             <i class="fa fa-paint-brush"></i>
                         </button>
                         <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
                             <li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
                             <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
                         </ul>
                         <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
                             <i class="fa fa-search"></i>
                         </button>
                     </div>

                     <h1 class="menu-title">Rust Compiler Development Guide</h1>

                     <div class="right-buttons">
                         <a href="../print.html" title="Print this book" aria-label="Print this book">
                             <i id="print-button" class="fa fa-print"></i>
                         </a>
                         <a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository">
                             <i id="git-repository-button" class="fa fa-github"></i>
                         </a>
                         <a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/tests/adding.md" title="Suggest an edit" aria-label="Suggest an edit">
                             <i id="git-edit-button" class="fa fa-edit"></i>
                         </a>

                     </div>
                 </div>

                 <div id="search-wrapper" class="hidden">
                     <form id="searchbar-outer" class="searchbar-outer">
                         <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
                     </form>
                     <div id="searchresults-outer" class="searchresults-outer hidden">
                         <div id="searchresults-header" class="searchresults-header"></div>
                         <ul id="searchresults">
                         </ul>
                     </div>
                 </div>

                 <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
                 <script>
                     document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
                     document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
                     Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
                         link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
                     });
                 </script>

                 <div id="content" class="content">
                     <main>
                         <h1 id="adding-new-tests"><a class="header" href="#adding-new-tests">Adding new tests</a></h1>
 <ul>
 <li><a href="#ui-test-walkthrough">UI test walkthrough</a>
 <ul>
 <li><a href="#step-1-add-a-test-file">Step 1: Add a test file</a></li>
 <li><a href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></li>
 <li><a href="#step-3-add-error-annotations">Step 3: Add error annotations</a></li>
 <li><a href="#step-4-review-the-output">Step 4: Review the output</a></li>
 <li><a href="#step-5-check-other-tests">Step 5: Check other tests</a></li>
 </ul>
 </li>
 <li><a href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></li>
 </ul>
 <p><strong>In general, we expect every PR that fixes a bug in rustc to come accompanied
 by a regression test of some kind.</strong> This test should fail in master but pass
 after the PR. These tests are really useful for preventing us from repeating the
 mistakes of the past.</p>
 <p>The first thing to decide is which kind of test to add. This will depend on the
 nature of the change and what you want to exercise. Here are some rough
 guidelines:</p>
 <ul>
 <li>The majority of compiler tests are done with <a href="compiletest.html">compiletest</a>.
 <ul>
 <li>The majority of compiletest tests are <a href="ui.html">UI</a> tests in the <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/"><code>tests/ui</code></a>
 directory.</li>
 </ul>
 </li>
 <li>Changes to the standard library are usually tested within the standard library
 itself.
 <ul>
 <li>The majority of standard library tests are written as doctests, which
 illustrate and exercise typical API behavior.</li>
 <li>Additional <a href="intro.html#package-tests">unit tests</a> should go in
 <code>library/${crate}/tests</code> (where <code>${crate}</code> is usually <code>core</code>, <code>alloc</code>, or
 <code>std</code>).</li>
 </ul>
 </li>
 <li>If the code is part of an isolated system, and you are not testing compiler
 output, consider using a <a href="intro.html#package-tests">unit or integration test</a>.</li>
 <li>Need to run rustdoc? Prefer a <code>rustdoc</code> or <code>rustdoc-ui</code> test. Occasionally
 you'll need <code>rustdoc-js</code> as well.</li>
 <li>Other compiletest test suites are generally used for special purposes:
 <ul>
 <li>Need to run gdb or lldb? Use the <code>debuginfo</code> test suite.</li>
 <li>Need to inspect LLVM IR or MIR IR? Use the <code>codegen</code> or <code>mir-opt</code> test
 suites.</li>
 <li>Need to inspect the resulting binary in some way? Or if all the other test
 suites are too limited for your purposes? Then use <code>run-make</code>.</li>
 <li>Check out the <a href="compiletest.html">compiletest</a> chapter for more specialized test suites.</li>
 </ul>
 </li>
 </ul>
 <p>After deciding on which kind of test to add, see <a href="best-practices.html">best
 practices</a> for guidance on how to author tests that are easy
 to work with that stand the test of time (i.e. if a test fails or need to be
 modified several years later, how can we make it easier for them?).</p>
 <h2 id="ui-test-walkthrough"><a class="header" href="#ui-test-walkthrough">UI test walkthrough</a></h2>
 <p>The following is a basic guide for creating a <a href="ui.html">UI test</a>, which is one of
 the most common compiler tests. For this tutorial, we'll be adding a test for an
 async error message.</p>
 <h3 id="step-1-add-a-test-file"><a class="header" href="#step-1-add-a-test-file">Step 1: Add a test file</a></h3>
 <p>The first step is to create a Rust source file somewhere in the <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/"><code>tests/ui</code></a>
 tree. When creating a test, do your best to find a good location and name (see
 <a href="ui.html#test-organization">Test organization</a> for more). Since naming is the
 hardest part of development, everything should be downhill from here!</p>
 <p>Let's place our async test at <code>tests/ui/async-await/await-without-async.rs</code>:</p>
 <pre><code class="language-rust ignore">// Provide diagnostics when the user writes `await` in a non-`async` function.
 //@ edition:2018

 async fn foo() {}

 fn bar() {
     foo().await
 }

 fn main() {}</code></pre>
 <p>A few things to notice about our test:</p>
 <ul>
 <li>The top should start with a short comment that <a href="#explanatory_comment">explains what the test is
 for</a>.</li>
 <li>The <code>//@ edition:2018</code> comment is called a <a href="directives.html">directive</a> which
 provides instructions to compiletest on how to build the test. Here we need to
 set the edition for <code>async</code> to work (the default is edition 2015).</li>
 <li>Following that is the source of the test. Try to keep it succinct and to the
 point. This may require some effort if you are trying to minimize an example
 from a bug report.</li>
 <li>We end this test with an empty <code>fn main</code> function. This is because the default
 for UI tests is a <code>bin</code> crate-type, and we don't want the "main not found"
 error in our test. Alternatively, you could add <code>#![crate_type="lib"]</code>.</li>
 </ul>
 <h3 id="step-2-generate-the-expected-output"><a class="header" href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></h3>
 <p>The next step is to create the expected output snapshots from the compiler. This
 can be done with the <code>--bless</code> option:</p>
 <pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs --bless
 </code></pre>
 <p>This will build the compiler (if it hasn't already been built), compile the
 test, and place the output of the compiler in a file called
 <code>tests/ui/async-await/await-without-async.stderr</code>.</p>
 <p>However, this step will fail! You should see an error message, something like
 this:</p>
 <blockquote>
 <p>error: /rust/tests/ui/async-await/await-without-async.rs:7: unexpected
 error: '7:10: 7:16: <code>await</code> is only allowed inside <code>async</code> functions and
 blocks E0728'</p>
 </blockquote>
 <p>This is because the stderr contains errors which were not matched by error
 annotations in the source file.</p>
 <h3 id="step-3-add-error-annotations"><a class="header" href="#step-3-add-error-annotations">Step 3: Add error annotations</a></h3>
 <p>Every error needs to be annotated with a comment in the source with the text of
 the error. In this case, we can add the following comment to our test file:</p>
 <pre><code class="language-rust ignore">fn bar() {
     foo().await
     //~^ ERROR `await` is only allowed inside `async` functions and blocks
 }</code></pre>
 <p>The <code>//~^</code> squiggle caret comment tells compiletest that the error belongs to
 the <em>previous</em> line (more on this in the <a href="ui.html#error-annotations">Error
 annotations</a> section).</p>
 <p>Save that, and run the test again:</p>
 <pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs
 </code></pre>
 <p>It should now pass, yay!</p>
 <h3 id="step-4-review-the-output"><a class="header" href="#step-4-review-the-output">Step 4: Review the output</a></h3>
 <p>Somewhat hand-in-hand with the previous step, you should inspect the <code>.stderr</code>
 file that was created to see if it looks like how you expect. If you are adding
 a new diagnostic message, now would be a good time to also consider how readable
 the message looks overall, particularly for people new to Rust.</p>
 <p>Our example <code>tests/ui/async-await/await-without-async.stderr</code> file should look
 like this:</p>
 <pre><code class="language-text">error[E0728]: `await` is only allowed inside `async` functions and blocks
   --&gt; $DIR/await-without-async.rs:7:10
    |
 LL | fn bar() {
    |    --- this is not `async`
 LL |     foo().await
    |          ^^^^^^ only allowed inside `async` functions and blocks

 error: aborting due to previous error

 For more information about this error, try `rustc --explain E0728`.
 </code></pre>
 <p>You may notice some things look a little different than the regular compiler
 output.</p>
 <ul>
 <li>The <code>$DIR</code> removes the path information which will differ between systems.</li>
 <li>The <code>LL</code> values replace the line numbers. That helps avoid small changes in
 the source from triggering large diffs. See the
 <a href="ui.html#normalization">Normalization</a> section for more.</li>
 </ul>
 <p>Around this stage, you may need to iterate over the last few steps a few times
 to tweak your test, re-bless the test, and re-review the output.</p>
 <h3 id="step-5-check-other-tests"><a class="header" href="#step-5-check-other-tests">Step 5: Check other tests</a></h3>
 <p>Sometimes when adding or changing a diagnostic message, this will affect other
 tests in the test suite. The final step before posting a PR is to check if you
 have affected anything else. Running the UI suite is usually a good start:</p>
 <pre><code class="language-sh">./x test tests/ui
 </code></pre>
 <p>If other tests start failing, you may need to investigate what has changed and
 if the new output makes sense.</p>
 <p>You may also need to re-bless the output with the <code>--bless</code> flag.</p>
 <p><a id="explanatory_comment"></a></p>
 <h2 id="comment-explaining-what-the-test-is-about"><a class="header" href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></h2>
 <p>The first comment of a test file should <strong>summarize the point of the test</strong>, and
 highlight what is important about it. If there is an issue number associated
 with the test, include the issue number.</p>
 <p>This comment doesn't have to be super extensive. Just something like "Regression
 test for #18060: match arms were matching in the wrong order." might already be
 enough.</p>
 <p>These comments are very useful to others later on when your test breaks, since
 they often can highlight what the problem is. They are also useful if for some
 reason the tests need to be refactored, since they let others know which parts
 of the test were important. Often a test must be rewritten because it no longer
 tests what it was meant to test, and then it's useful to know what it <em>was</em>
 meant to test exactly.</p>

                     </main>

                     <nav class="nav-wrapper" aria-label="Page navigation">
                         <!-- Mobile navigation buttons -->
                             <a rel="prev" href="../tests/ci.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
                                 <i class="fa fa-angle-left"></i>
                             </a>

                             <a rel="next prefetch" href="../tests/best-practices.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
                                 <i class="fa fa-angle-right"></i>
                             </a>

                         <div style="clear: both"></div>
                     </nav>
                 </div>
             </div>

             <nav class="nav-wide-wrapper" aria-label="Page navigation">
                     <a rel="prev" href="../tests/ci.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
                         <i class="fa fa-angle-left"></i>
                     </a>

                     <a rel="next prefetch" href="../tests/best-practices.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
                         <i class="fa fa-angle-right"></i>
                     </a>
             </nav>

         </div>


         <script>
             window.playground_copyable = true;
         </script>


         <script src="../elasticlunr.min.js"></script>
         <script src="../mark.min.js"></script>
         <script src="../searcher.js"></script>

         <script src="../clipboard.min.js"></script>
         <script src="../highlight.js"></script>
         <script src="../book.js"></script>

         <!-- Custom JS scripts -->
         <script src="../mermaid.min.js"></script>
         <script src="../mermaid-init.js"></script>


     </div>
     </body>
 </html>
	<!DOCTYPE HTML>
	<html lang="en" class="light sidebar-visible" dir="ltr">
	<head>
	<!-- Book generated using mdBook -->
	<meta charset="UTF-8">
	<title>Adding new tests - Rust Compiler Development Guide</title>


	<!-- Custom HTML head -->

	<meta name="description" content="A guide to developing the Rust compiler (rustc)">
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<meta name="theme-color" content="#ffffff">

	<link rel="icon" href="../favicon.svg">
	<link rel="shortcut icon" href="../favicon.png">
	<link rel="stylesheet" href="../css/variables.css">
	<link rel="stylesheet" href="../css/general.css">
	<link rel="stylesheet" href="../css/chrome.css">
	<link rel="stylesheet" href="../css/print.css" media="print">

	<!-- Fonts -->
	<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
	<link rel="stylesheet" href="../fonts/fonts.css">

	<!-- Highlight.js Stylesheets -->
	<link rel="stylesheet" id="highlight-css" href="../highlight.css">
	<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
	<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">

	<!-- Custom theme stylesheets -->


	<!-- Provide site root and default themes to javascript -->
	<script>
	const path_to_root = "../";
	const default_light_theme = "light";
	const default_dark_theme = "navy";
	</script>
	<!-- Start loading toc.js asap -->
	<script src="../toc.js"></script>
	</head>
	<body>
	<div id="body-container">
	<!-- Work around some values being stored in localStorage wrapped in quotes -->
	<script>
	try {
	let theme = localStorage.getItem('mdbook-theme');
	let sidebar = localStorage.getItem('mdbook-sidebar');

	if (theme.startsWith('"') && theme.endsWith('"')) {
	localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
	}

	if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
	localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
	}
	} catch (e) { }
	</script>

	<!-- Set the theme before any content is loaded, prevents flash -->
	<script>
	const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
	let theme;
	try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
	if (theme === null \|\| theme === undefined) { theme = default_theme; }
	const html = document.documentElement;
	html.classList.remove('light')
	html.classList.add(theme);
	html.classList.add("js");
	</script>

	<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">

	<!-- Hide / unhide sidebar before it is displayed -->
	<script>
	let sidebar = null;
	const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
	if (document.body.clientWidth >= 1080) {
	try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
	sidebar = sidebar \|\| 'visible';
	} else {
	sidebar = 'hidden';
	}
	sidebar_toggle.checked = sidebar === 'visible';
	html.classList.remove('sidebar-visible');
	html.classList.add("sidebar-" + sidebar);
	</script>

	<nav id="sidebar" class="sidebar" aria-label="Table of contents">
	<!-- populated by js -->
	<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
	<noscript>
	<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
	</noscript>
	<div id="sidebar-resize-handle" class="sidebar-resize-handle">
	<div class="sidebar-resize-indicator"></div>
	</div>
	</nav>

	<div id="page-wrapper" class="page-wrapper">

	<div class="page">
	<div id="menu-bar-hover-placeholder"></div>
	<div id="menu-bar" class="menu-bar sticky">
	<div class="left-buttons">
	<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
	<i class="fa fa-bars"></i>
	</label>
	<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
	<i class="fa fa-paint-brush"></i>
	</button>
	<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
	<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
	<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
	<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
	<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
	<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
	<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
	</ul>
	<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
	<i class="fa fa-search"></i>
	</button>
	</div>

	<h1 class="menu-title">Rust Compiler Development Guide</h1>

	<div class="right-buttons">
	<a href="../print.html" title="Print this book" aria-label="Print this book">
	<i id="print-button" class="fa fa-print"></i>
	</a>
	<a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository">
	<i id="git-repository-button" class="fa fa-github"></i>
	</a>
	<a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/tests/adding.md" title="Suggest an edit" aria-label="Suggest an edit">
	<i id="git-edit-button" class="fa fa-edit"></i>
	</a>

	</div>
	</div>

	<div id="search-wrapper" class="hidden">
	<form id="searchbar-outer" class="searchbar-outer">
	<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
	</form>
	<div id="searchresults-outer" class="searchresults-outer hidden">
	<div id="searchresults-header" class="searchresults-header"></div>
	<ul id="searchresults">
	</ul>
	</div>
	</div>

	<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
	<script>
	document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
	document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
	Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
	link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
	});
	</script>

	<div id="content" class="content">
	<main>
	<h1 id="adding-new-tests"><a class="header" href="#adding-new-tests">Adding new tests</a></h1>
	<ul>
	<li><a href="#ui-test-walkthrough">UI test walkthrough</a>
	<ul>
	<li><a href="#step-1-add-a-test-file">Step 1: Add a test file</a></li>
	<li><a href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></li>
	<li><a href="#step-3-add-error-annotations">Step 3: Add error annotations</a></li>
	<li><a href="#step-4-review-the-output">Step 4: Review the output</a></li>
	<li><a href="#step-5-check-other-tests">Step 5: Check other tests</a></li>
	</ul>
	</li>
	<li><a href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></li>
	</ul>
	<p><strong>In general, we expect every PR that fixes a bug in rustc to come accompanied
	by a regression test of some kind.</strong> This test should fail in master but pass
	after the PR. These tests are really useful for preventing us from repeating the
	mistakes of the past.</p>
	<p>The first thing to decide is which kind of test to add. This will depend on the
	nature of the change and what you want to exercise. Here are some rough
	guidelines:</p>
	<ul>
	<li>The majority of compiler tests are done with <a href="compiletest.html">compiletest</a>.
	<ul>
	<li>The majority of compiletest tests are <a href="ui.html">UI</a> tests in the <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/"><code>tests/ui</code></a>
	directory.</li>
	</ul>
	</li>
	<li>Changes to the standard library are usually tested within the standard library
	itself.
	<ul>
	<li>The majority of standard library tests are written as doctests, which
	illustrate and exercise typical API behavior.</li>
	<li>Additional <a href="intro.html#package-tests">unit tests</a> should go in
	<code>library/${crate}/tests</code> (where <code>${crate}</code> is usually <code>core</code>, <code>alloc</code>, or
	<code>std</code>).</li>
	</ul>
	</li>
	<li>If the code is part of an isolated system, and you are not testing compiler
	output, consider using a <a href="intro.html#package-tests">unit or integration test</a>.</li>
	<li>Need to run rustdoc? Prefer a <code>rustdoc</code> or <code>rustdoc-ui</code> test. Occasionally
	you'll need <code>rustdoc-js</code> as well.</li>
	<li>Other compiletest test suites are generally used for special purposes:
	<ul>
	<li>Need to run gdb or lldb? Use the <code>debuginfo</code> test suite.</li>
	<li>Need to inspect LLVM IR or MIR IR? Use the <code>codegen</code> or <code>mir-opt</code> test
	suites.</li>
	<li>Need to inspect the resulting binary in some way? Or if all the other test
	suites are too limited for your purposes? Then use <code>run-make</code>.</li>
	<li>Check out the <a href="compiletest.html">compiletest</a> chapter for more specialized test suites.</li>
	</ul>
	</li>
	</ul>
	<p>After deciding on which kind of test to add, see <a href="best-practices.html">best
	practices</a> for guidance on how to author tests that are easy
	to work with that stand the test of time (i.e. if a test fails or need to be
	modified several years later, how can we make it easier for them?).</p>
	<h2 id="ui-test-walkthrough"><a class="header" href="#ui-test-walkthrough">UI test walkthrough</a></h2>
	<p>The following is a basic guide for creating a <a href="ui.html">UI test</a>, which is one of
	the most common compiler tests. For this tutorial, we'll be adding a test for an
	async error message.</p>
	<h3 id="step-1-add-a-test-file"><a class="header" href="#step-1-add-a-test-file">Step 1: Add a test file</a></h3>
	<p>The first step is to create a Rust source file somewhere in the <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/"><code>tests/ui</code></a>
	tree. When creating a test, do your best to find a good location and name (see
	<a href="ui.html#test-organization">Test organization</a> for more). Since naming is the
	hardest part of development, everything should be downhill from here!</p>
	<p>Let's place our async test at <code>tests/ui/async-await/await-without-async.rs</code>:</p>
	<pre><code class="language-rust ignore">// Provide diagnostics when the user writes `await` in a non-`async` function.
	//@ edition:2018

	async fn foo() {}

	fn bar() {
	foo().await
	}

	fn main() {}</code></pre>
	<p>A few things to notice about our test:</p>
	<ul>
	<li>The top should start with a short comment that <a href="#explanatory_comment">explains what the test is
	for</a>.</li>
	<li>The <code>//@ edition:2018</code> comment is called a <a href="directives.html">directive</a> which
	provides instructions to compiletest on how to build the test. Here we need to
	set the edition for <code>async</code> to work (the default is edition 2015).</li>
	<li>Following that is the source of the test. Try to keep it succinct and to the
	point. This may require some effort if you are trying to minimize an example
	from a bug report.</li>
	<li>We end this test with an empty <code>fn main</code> function. This is because the default
	for UI tests is a <code>bin</code> crate-type, and we don't want the "main not found"
	error in our test. Alternatively, you could add <code>#![crate_type="lib"]</code>.</li>
	</ul>
	<h3 id="step-2-generate-the-expected-output"><a class="header" href="#step-2-generate-the-expected-output">Step 2: Generate the expected output</a></h3>
	<p>The next step is to create the expected output snapshots from the compiler. This
	can be done with the <code>--bless</code> option:</p>
	<pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs --bless
	</code></pre>
	<p>This will build the compiler (if it hasn't already been built), compile the
	test, and place the output of the compiler in a file called
	<code>tests/ui/async-await/await-without-async.stderr</code>.</p>
	<p>However, this step will fail! You should see an error message, something like
	this:</p>
	<blockquote>
	<p>error: /rust/tests/ui/async-await/await-without-async.rs:7: unexpected
	error: '7:10: 7:16: <code>await</code> is only allowed inside <code>async</code> functions and
	blocks E0728'</p>
	</blockquote>
	<p>This is because the stderr contains errors which were not matched by error
	annotations in the source file.</p>
	<h3 id="step-3-add-error-annotations"><a class="header" href="#step-3-add-error-annotations">Step 3: Add error annotations</a></h3>
	<p>Every error needs to be annotated with a comment in the source with the text of
	the error. In this case, we can add the following comment to our test file:</p>
	<pre><code class="language-rust ignore">fn bar() {
	foo().await
	//~^ ERROR `await` is only allowed inside `async` functions and blocks
	}</code></pre>
	<p>The <code>//~^</code> squiggle caret comment tells compiletest that the error belongs to
	the <em>previous</em> line (more on this in the <a href="ui.html#error-annotations">Error
	annotations</a> section).</p>
	<p>Save that, and run the test again:</p>
	<pre><code class="language-sh">./x test tests/ui/async-await/await-without-async.rs
	</code></pre>
	<p>It should now pass, yay!</p>
	<h3 id="step-4-review-the-output"><a class="header" href="#step-4-review-the-output">Step 4: Review the output</a></h3>
	<p>Somewhat hand-in-hand with the previous step, you should inspect the <code>.stderr</code>
	file that was created to see if it looks like how you expect. If you are adding
	a new diagnostic message, now would be a good time to also consider how readable
	the message looks overall, particularly for people new to Rust.</p>
	<p>Our example <code>tests/ui/async-await/await-without-async.stderr</code> file should look
	like this:</p>
	<pre><code class="language-text">error[E0728]: `await` is only allowed inside `async` functions and blocks
	--> $DIR/await-without-async.rs:7:10
	\|
	LL \| fn bar() {
	\| --- this is not `async`
	LL \| foo().await
	\| ^^^^^^ only allowed inside `async` functions and blocks

	error: aborting due to previous error

	For more information about this error, try `rustc --explain E0728`.
	</code></pre>
	<p>You may notice some things look a little different than the regular compiler
	output.</p>
	<ul>
	<li>The <code>$DIR</code> removes the path information which will differ between systems.</li>
	<li>The <code>LL</code> values replace the line numbers. That helps avoid small changes in
	the source from triggering large diffs. See the
	<a href="ui.html#normalization">Normalization</a> section for more.</li>
	</ul>
	<p>Around this stage, you may need to iterate over the last few steps a few times
	to tweak your test, re-bless the test, and re-review the output.</p>
	<h3 id="step-5-check-other-tests"><a class="header" href="#step-5-check-other-tests">Step 5: Check other tests</a></h3>
	<p>Sometimes when adding or changing a diagnostic message, this will affect other
	tests in the test suite. The final step before posting a PR is to check if you
	have affected anything else. Running the UI suite is usually a good start:</p>
	<pre><code class="language-sh">./x test tests/ui
	</code></pre>
	<p>If other tests start failing, you may need to investigate what has changed and
	if the new output makes sense.</p>
	<p>You may also need to re-bless the output with the <code>--bless</code> flag.</p>
	<p><a id="explanatory_comment"></a></p>
	<h2 id="comment-explaining-what-the-test-is-about"><a class="header" href="#comment-explaining-what-the-test-is-about">Comment explaining what the test is about</a></h2>
	<p>The first comment of a test file should <strong>summarize the point of the test</strong>, and
	highlight what is important about it. If there is an issue number associated
	with the test, include the issue number.</p>
	<p>This comment doesn't have to be super extensive. Just something like "Regression
	test for #18060: match arms were matching in the wrong order." might already be
	enough.</p>
	<p>These comments are very useful to others later on when your test breaks, since
	they often can highlight what the problem is. They are also useful if for some
	reason the tests need to be refactored, since they let others know which parts
	of the test were important. Often a test must be rewritten because it no longer
	tests what it was meant to test, and then it's useful to know what it <em>was</em>
	meant to test exactly.</p>

	</main>

	<nav class="nav-wrapper" aria-label="Page navigation">
	<!-- Mobile navigation buttons -->
	<a rel="prev" href="../tests/ci.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
	<i class="fa fa-angle-left"></i>
	</a>

	<a rel="next prefetch" href="../tests/best-practices.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
	<i class="fa fa-angle-right"></i>
	</a>

	<div style="clear: both"></div>
	</nav>
	</div>
	</div>

	<nav class="nav-wide-wrapper" aria-label="Page navigation">
	<a rel="prev" href="../tests/ci.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
	<i class="fa fa-angle-left"></i>
	</a>

	<a rel="next prefetch" href="../tests/best-practices.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
	<i class="fa fa-angle-right"></i>
	</a>
	</nav>

	</div>




	<script>
	window.playground_copyable = true;
	</script>


	<script src="../elasticlunr.min.js"></script>
	<script src="../mark.min.js"></script>
	<script src="../searcher.js"></script>

	<script src="../clipboard.min.js"></script>
	<script src="../highlight.js"></script>
	<script src="../book.js"></script>

	<!-- Custom JS scripts -->
	<script src="../mermaid.min.js"></script>
	<script src="../mermaid-init.js"></script>


	</div>
	</body>
	</html>