| <!DOCTYPE HTML> |
| <html lang="en" class="light sidebar-visible" dir="ltr"> |
| <head> |
| <!-- Book generated using mdBook --> |
| <meta charset="UTF-8"> |
| <title>UI tests - Rust Compiler Development Guide</title> |
| |
| |
| <!-- Custom HTML head --> |
| |
| <meta name="description" content="A guide to developing the Rust compiler (rustc)"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <meta name="theme-color" content="#ffffff"> |
| |
| <link rel="icon" href="../favicon.svg"> |
| <link rel="shortcut icon" href="../favicon.png"> |
| <link rel="stylesheet" href="../css/variables.css"> |
| <link rel="stylesheet" href="../css/general.css"> |
| <link rel="stylesheet" href="../css/chrome.css"> |
| <link rel="stylesheet" href="../css/print.css" media="print"> |
| |
| <!-- Fonts --> |
| <link rel="stylesheet" href="../FontAwesome/css/font-awesome.css"> |
| <link rel="stylesheet" href="../fonts/fonts.css"> |
| |
| <!-- Highlight.js Stylesheets --> |
| <link rel="stylesheet" id="highlight-css" href="../highlight.css"> |
| <link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css"> |
| <link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css"> |
| |
| <!-- Custom theme stylesheets --> |
| |
| |
| <!-- Provide site root and default themes to javascript --> |
| <script> |
| const path_to_root = "../"; |
| const default_light_theme = "light"; |
| const default_dark_theme = "navy"; |
| </script> |
| <!-- Start loading toc.js asap --> |
| <script src="../toc.js"></script> |
| </head> |
| <body> |
| <div id="body-container"> |
| <!-- Work around some values being stored in localStorage wrapped in quotes --> |
| <script> |
| try { |
| let theme = localStorage.getItem('mdbook-theme'); |
| let sidebar = localStorage.getItem('mdbook-sidebar'); |
| |
| if (theme.startsWith('"') && theme.endsWith('"')) { |
| localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1)); |
| } |
| |
| if (sidebar.startsWith('"') && sidebar.endsWith('"')) { |
| localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1)); |
| } |
| } catch (e) { } |
| </script> |
| |
| <!-- Set the theme before any content is loaded, prevents flash --> |
| <script> |
| const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme; |
| let theme; |
| try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { } |
| if (theme === null || theme === undefined) { theme = default_theme; } |
| const html = document.documentElement; |
| html.classList.remove('light') |
| html.classList.add(theme); |
| html.classList.add("js"); |
| </script> |
| |
| <input type="checkbox" id="sidebar-toggle-anchor" class="hidden"> |
| |
| <!-- Hide / unhide sidebar before it is displayed --> |
| <script> |
| let sidebar = null; |
| const sidebar_toggle = document.getElementById("sidebar-toggle-anchor"); |
| if (document.body.clientWidth >= 1080) { |
| try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { } |
| sidebar = sidebar || 'visible'; |
| } else { |
| sidebar = 'hidden'; |
| } |
| sidebar_toggle.checked = sidebar === 'visible'; |
| html.classList.remove('sidebar-visible'); |
| html.classList.add("sidebar-" + sidebar); |
| </script> |
| |
| <nav id="sidebar" class="sidebar" aria-label="Table of contents"> |
| <!-- populated by js --> |
| <mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox> |
| <noscript> |
| <iframe class="sidebar-iframe-outer" src="../toc.html"></iframe> |
| </noscript> |
| <div id="sidebar-resize-handle" class="sidebar-resize-handle"> |
| <div class="sidebar-resize-indicator"></div> |
| </div> |
| </nav> |
| |
| <div id="page-wrapper" class="page-wrapper"> |
| |
| <div class="page"> |
| <div id="menu-bar-hover-placeholder"></div> |
| <div id="menu-bar" class="menu-bar sticky"> |
| <div class="left-buttons"> |
| <label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar"> |
| <i class="fa fa-bars"></i> |
| </label> |
| <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list"> |
| <i class="fa fa-paint-brush"></i> |
| </button> |
| <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu"> |
| <li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="light">Light</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li> |
| <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li> |
| </ul> |
| <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar"> |
| <i class="fa fa-search"></i> |
| </button> |
| </div> |
| |
| <h1 class="menu-title">Rust Compiler Development Guide</h1> |
| |
| <div class="right-buttons"> |
| <a href="../print.html" title="Print this book" aria-label="Print this book"> |
| <i id="print-button" class="fa fa-print"></i> |
| </a> |
| <a href="https://github.com/rust-lang/rustc-dev-guide" title="Git repository" aria-label="Git repository"> |
| <i id="git-repository-button" class="fa fa-github"></i> |
| </a> |
| <a href="https://github.com/rust-lang/rustc-dev-guide/edit/master/src/tests/ui.md" title="Suggest an edit" aria-label="Suggest an edit"> |
| <i id="git-edit-button" class="fa fa-edit"></i> |
| </a> |
| |
| </div> |
| </div> |
| |
| <div id="search-wrapper" class="hidden"> |
| <form id="searchbar-outer" class="searchbar-outer"> |
| <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header"> |
| </form> |
| <div id="searchresults-outer" class="searchresults-outer hidden"> |
| <div id="searchresults-header" class="searchresults-header"></div> |
| <ul id="searchresults"> |
| </ul> |
| </div> |
| </div> |
| |
| <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM --> |
| <script> |
| document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible'); |
| document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible'); |
| Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) { |
| link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1); |
| }); |
| </script> |
| |
| <div id="content" class="content"> |
| <main> |
| <h1 id="ui-tests"><a class="header" href="#ui-tests">UI tests</a></h1> |
| <ul> |
| <li><a href="#introduction">Introduction</a></li> |
| <li><a href="#general-structure-of-a-test">General structure of a test</a></li> |
| <li><a href="#output-comparison">Output comparison</a> |
| <ul> |
| <li><a href="#normalization">Normalization</a></li> |
| </ul> |
| </li> |
| <li><a href="#error-annotations">Error annotations</a> |
| <ul> |
| <li><a href="#error-annotation-examples">Error annotation examples</a> |
| <ul> |
| <li><a href="#positioned-on-error-line">Positioned on error line</a></li> |
| <li><a href="#positioned-below-error-line">Positioned below error line</a></li> |
| <li><a href="#use-same-error-line-as-defined-on-error-annotation-line-above">Use same error line as defined on error annotation line above</a></li> |
| <li><a href="#positioned-above-error-line">Positioned above error line</a></li> |
| <li><a href="#error-without-line-information">Error without line information</a></li> |
| </ul> |
| </li> |
| <li><a href="#error-pattern"><code>error-pattern</code></a></li> |
| <li><a href="#diagnostic-kinds-error-levels">Diagnostic kinds (error levels)</a></li> |
| <li><a href="#cfg-revisions"><code>cfg</code> revisions</a></li> |
| </ul> |
| </li> |
| <li><a href="#controlling-passfail-expectations">Controlling pass/fail expectations</a></li> |
| <li><a href="#known-bugs">Known bugs</a></li> |
| <li><a href="#test-organization">Test organization</a></li> |
| <li><a href="#rustfix-tests">Rustfix tests</a></li> |
| <li><a href="#compare-modes">Compare modes</a></li> |
| <li><a href="#rustc_-test-attributes"><code>rustc_*</code> TEST attributes</a></li> |
| <li><a href="#ui-test-mode-preset-lint-levels">UI test mode preset lint levels</a></li> |
| </ul> |
| <p>UI tests are a particular <a href="compiletest.html#test-suites">test suite</a> of |
| compiletest.</p> |
| <h2 id="introduction"><a class="header" href="#introduction">Introduction</a></h2> |
| <p>The tests in <a href="https://github.com/rust-lang/rust/blob/master/tests/ui"><code>tests/ui</code></a> are a collection of general-purpose tests which |
| primarily focus on validating the console output of the compiler, but can be |
| used for many other purposes. For example, tests can also be configured to <a href="#controlling-passfail-expectations">run |
| the resulting program</a> to verify its |
| behavior.</p> |
| <p>For a survey of each subdirectory's purpose under <code>tests/ui</code>, consult the |
| <a href="https://github.com/rust-lang/rust/tree/master/tests/ui/SUMMARY.md">SUMMARY.md</a>. |
| This is useful if you write a new test, and are looking for a category to |
| place it in.</p> |
| <p>If you need to work with <code>#![no_std]</code> cross-compiling tests, consult the |
| <a href="./minicore.html"><code>minicore</code> test auxiliary</a> chapter.</p> |
| <h2 id="general-structure-of-a-test"><a class="header" href="#general-structure-of-a-test">General structure of a test</a></h2> |
| <p>A test consists of a Rust source file located anywhere in the <code>tests/ui</code> |
| directory, but they should be placed in a suitable sub-directory. For example, |
| <a href="https://github.com/rust-lang/rust/blob/master/tests/ui/hello.rs"><code>tests/ui/hello.rs</code></a> is a basic hello-world test.</p> |
| <p>Compiletest will use <code>rustc</code> to compile the test, and compare the output against |
| the expected output which is stored in a <code>.stdout</code> or <code>.stderr</code> file located |
| next to the test. See <a href="#output-comparison">Output comparison</a> for more.</p> |
| <p>Additionally, errors and warnings should be annotated with comments within the |
| source file. See <a href="#error-annotations">Error annotations</a> for more.</p> |
| <p>Compiletest <a href="directives.html">directives</a> in the form of special comments prefixed |
| with <code>//@</code> control how the test is compiled and what the expected behavior is.</p> |
| <p>Tests are expected to fail to compile, since most tests are testing compiler |
| errors. You can change that behavior with a directive, see <a href="#controlling-passfail-expectations">Controlling |
| pass/fail expectations</a>.</p> |
| <p>By default, a test is built as an executable binary. If you need a different |
| crate type, you can use the <code>#![crate_type]</code> attribute to set it as needed.</p> |
| <h2 id="output-comparison"><a class="header" href="#output-comparison">Output comparison</a></h2> |
| <p>UI tests store the expected output from the compiler in <code>.stderr</code> and <code>.stdout</code> |
| snapshots next to the test. You normally generate these files with the <code>--bless</code> |
| CLI option, and then inspect them manually to verify they contain what you |
| expect.</p> |
| <p>The output is normalized to ignore unwanted differences, see the |
| <a href="#normalization">Normalization</a> section. If the file is missing, then |
| compiletest expects the corresponding output to be empty.</p> |
| <p>A common reason to use normalization, revisions, and most of the other following tools, |
| is to account for platform differences. Consider alternatives to these tools, like |
| e.g. using the <code>extern "rust-invalid"</code> ABI that is invalid on every platform |
| instead of fixing the test to use cross-compilation and testing every possibly-invalid ABI.</p> |
| <p>There can be multiple stdout/stderr files. The general form is:</p> |
| <pre><code class="language-text">*test-name*`.`*revision*`.`*compare_mode*`.`*extension* |
| </code></pre> |
| <ul> |
| <li><em>test-name</em> cannot contain dots. This is so that the general form of test |
| output filenames have a predictable form we can pattern match on in order to |
| track stray test output files.</li> |
| <li><em>revision</em> is the <a href="#cfg-revisions">revision</a> name. This is not included when |
| not using revisions.</li> |
| <li><em>compare_mode</em> is the <a href="#compare-modes">compare mode</a>. This will only be |
| checked when the given compare mode is active. If the file does not exist, |
| then compiletest will check for a file without the compare mode.</li> |
| <li><em>extension</em> is the kind of output being checked: |
| <ul> |
| <li><code>stderr</code> — compiler stderr</li> |
| <li><code>stdout</code> — compiler stdout</li> |
| <li><code>run.stderr</code> — stderr when running the test</li> |
| <li><code>run.stdout</code> — stdout when running the test</li> |
| <li><code>64bit.stderr</code> — compiler stderr with <code>stderr-per-bitwidth</code> directive on a |
| 64-bit target</li> |
| <li><code>32bit.stderr</code> — compiler stderr with <code>stderr-per-bitwidth</code> directive on a |
| 32-bit target</li> |
| </ul> |
| </li> |
| </ul> |
| <p>A simple example would be <code>foo.stderr</code> next to a <code>foo.rs</code> test. |
| A more complex example would be <code>foo.my-revision.polonius.stderr</code>.</p> |
| <p>There are several <a href="directives.html">directives</a> which will change how compiletest |
| will check for output files:</p> |
| <ul> |
| <li><code>stderr-per-bitwidth</code> — checks separate output files based on the target |
| pointer width. Consider using the <code>normalize-stderr</code> directive instead (see |
| <a href="#normalization">Normalization</a>).</li> |
| <li><code>dont-check-compiler-stderr</code> — Ignores stderr from the compiler.</li> |
| <li><code>dont-check-compiler-stdout</code> — Ignores stdout from the compiler.</li> |
| </ul> |
| <p>UI tests run with <code>-Zdeduplicate-diagnostics=no</code> flag which disables rustc's |
| built-in diagnostic deduplication mechanism. This means you may see some |
| duplicate messages in the output. This helps illuminate situations where |
| duplicate diagnostics are being generated.</p> |
| <h3 id="normalization"><a class="header" href="#normalization">Normalization</a></h3> |
| <p>The compiler output is normalized to eliminate output difference between |
| platforms, mainly about filenames.</p> |
| <p>Compiletest makes the following replacements on the compiler output:</p> |
| <ul> |
| <li>The directory where the test is defined is replaced with <code>$DIR</code>. Example: |
| <code>/path/to/rust/tests/ui/error-codes</code></li> |
| <li>The directory to the standard library source is replaced with <code>$SRC_DIR</code>. |
| Example: <code>/path/to/rust/library</code></li> |
| <li>Line and column numbers for paths in <code>$SRC_DIR</code> are replaced with <code>LL:COL</code>. |
| This helps ensure that changes to the layout of the standard library do not |
| cause widespread changes to the <code>.stderr</code> files. Example: |
| <code>$SRC_DIR/alloc/src/sync.rs:53:46</code></li> |
| <li>The base directory where the test's output goes is replaced with |
| <code>$TEST_BUILD_DIR</code>. This only comes up in a few rare circumstances. Example: |
| <code>/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui</code></li> |
| <li>The real directory to the standard library source is replaced with <code>$SRC_DIR_REAL</code>.</li> |
| <li>The real directory to the compiler source is replaced with <code>$COMPILER_DIR_REAL</code>.</li> |
| <li>Tabs are replaced with <code>\t</code>.</li> |
| <li>Backslashes (<code>\</code>) are converted to forward slashes (<code>/</code>) within paths (using a |
| heuristic). This helps normalize differences with Windows-style paths.</li> |
| <li>CRLF newlines are converted to LF.</li> |
| <li>Error line annotations like <code>//~ ERROR some message</code> are removed.</li> |
| <li>Various v0 and legacy symbol hashes are replaced with placeholders like |
| <code>[HASH]</code> or <code><SYMBOL_HASH></code>.</li> |
| </ul> |
| <p>Additionally, the compiler is run with the <code>-Z ui-testing</code> flag which causes |
| the compiler itself to apply some changes to the diagnostic output to make it |
| more suitable for UI testing.</p> |
| <p>For example, it will anonymize line numbers in the output (line numbers |
| prefixing each source line are replaced with <code>LL</code>). In extremely rare |
| situations, this mode can be disabled with the directive <code>//@ compile-flags: -Z ui-testing=no</code>.</p> |
| <p>Note: The line and column numbers for <code>--></code> lines pointing to the test are <em>not</em> |
| normalized, and left as-is. This ensures that the compiler continues to point to |
| the correct location, and keeps the stderr files readable. Ideally all |
| line/column information would be retained, but small changes to the source |
| causes large diffs, and more frequent merge conflicts and test errors.</p> |
| <p>Sometimes these built-in normalizations are not enough. In such cases, you may |
| provide custom normalization rules using <code>normalize-*</code> directives, e.g.</p> |
| <pre><code class="language-rust ignore">//@ normalize-stdout: "foo" -> "bar" |
| //@ normalize-stderr: "foo" -> "bar" |
| //@ normalize-stderr-32bit: "fn\(\) \(32 bits\)" -> "fn\(\) \($$PTR bits\)" |
| //@ normalize-stderr-64bit: "fn\(\) \(64 bits\)" -> "fn\(\) \($$PTR bits\)"</code></pre> |
| <p>This tells the test, on 32-bit platforms, whenever the compiler writes <code>fn() (32 bits)</code> to stderr, it should be normalized to read <code>fn() ($PTR bits)</code> instead. |
| Similar for 64-bit. The replacement is performed by regexes using default regex |
| flavor provided by <code>regex</code> crate.</p> |
| <p>The corresponding reference file will use the normalized output to test both |
| 32-bit and 64-bit platforms:</p> |
| <pre><code class="language-text">... |
| | |
| = note: source type: fn() ($PTR bits) |
| = note: target type: u16 (16 bits) |
| ... |
| </code></pre> |
| <p>Please see <a href="https://github.com/rust-lang/rust/blob/master/tests/ui/transmute/main.rs"><code>ui/transmute/main.rs</code></a> and <a href="https://github.com/rust-lang/rust/blob/master/tests/ui/transmute/main.stderr"><code>main.stderr</code></a> for a concrete |
| usage example.</p> |
| <h2 id="error-annotations"><a class="header" href="#error-annotations">Error annotations</a></h2> |
| <p>Error annotations specify the errors that the compiler is expected to emit. They |
| are "attached" to the line in source where the error is located.</p> |
| <pre><code class="language-rust ignore">fn main() { |
| boom //~ ERROR cannot find value `boom` in this scope [E0425] |
| }</code></pre> |
| <p>Although UI tests have a <code>.stderr</code> file which contains the entire compiler |
| output, UI tests require that errors are also annotated within the source. This |
| redundancy helps avoid mistakes since the <code>.stderr</code> files are usually |
| auto-generated. It also helps to directly see where the error spans are expected |
| to point to by looking at one file instead of having to compare the <code>.stderr</code> |
| file with the source. Finally, they ensure that no additional unexpected errors |
| are generated.</p> |
| <p>They have several forms, but generally are a comment with the diagnostic level |
| (such as <code>ERROR</code>) and a substring of the expected error output. You don't have |
| to write out the entire message, just make sure to include the important part of |
| the message to make it self-documenting.</p> |
| <p>Most error annotations need to match with the line of the diagnostic. There are |
| several ways to match the message with the line (see the examples below):</p> |
| <ul> |
| <li><code>~</code>: Associates the error level and message with the <em>current</em> line</li> |
| <li><code>~^</code>: Associates the error level and message with the <em>previous</em> error |
| annotation line. Each caret (<code>^</code>) that you add adds a line to this, so <code>~^^^</code> |
| is three lines above the error annotation line.</li> |
| <li><code>~|</code>: Associates the error level and message with the <em>same</em> line as the |
| <em>previous comment</em>. This is more convenient than using multiple carets when |
| there are multiple messages associated with the same line.</li> |
| <li><code>~v</code>: Associates the error level and message with the <em>next</em> error |
| annotation line. Each symbol (<code>v</code>) that you add adds a line to this, so <code>~vvv</code> |
| is three lines below the error annotation line.</li> |
| </ul> |
| <p>Example:</p> |
| <pre><code class="language-rust ignore">let _ = same_line; //~ ERROR undeclared variable |
| fn meow(_: [u8]) {} |
| //~^ ERROR unsized |
| //~| ERROR anonymous parameters</code></pre> |
| <p>The space character between <code>//~</code> (or other variants) and the subsequent text is |
| negligible (i.e. there is no semantic difference between <code>//~ ERROR</code> and |
| <code>//~ERROR</code> although the former is more common in the codebase).</p> |
| <p><code>~? <diagnostic kind></code> (example being <code>~? ERROR</code>) |
| is used to match diagnostics <em>without</em> line info at all, |
| or where the line info is outside the main test file<sup class="footnote-reference" id="fr-main test file-1"><a href="#footnote-main test file">1</a></sup>. |
| These annotations can be placed on any line in the test file.</p> |
| <h3 id="error-annotation-examples"><a class="header" href="#error-annotation-examples">Error annotation examples</a></h3> |
| <p>Here are examples of error annotations on different lines of UI test source.</p> |
| <h4 id="positioned-on-error-line"><a class="header" href="#positioned-on-error-line">Positioned on error line</a></h4> |
| <p>Use the <code>//~ ERROR</code> idiom:</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let x = (1, 2, 3); |
| match x { |
| (_a, _x @ ..) => {} //~ ERROR `_x @` is not allowed in a tuple |
| _ => {} |
| } |
| }</code></pre> |
| <h4 id="positioned-below-error-line"><a class="header" href="#positioned-below-error-line">Positioned below error line</a></h4> |
| <p>Use the <code>//~^</code> idiom with number of carets in the string to indicate the number |
| of lines above. In the example below, the error line is four lines above the |
| error annotation line so four carets are included in the annotation.</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let x = (1, 2, 3); |
| match x { |
| (_a, _x @ ..) => {} // <- the error is on this line |
| _ => {} |
| } |
| } |
| //~^^^^ ERROR `_x @` is not allowed in a tuple</code></pre> |
| <h4 id="use-same-error-line-as-defined-on-error-annotation-line-above"><a class="header" href="#use-same-error-line-as-defined-on-error-annotation-line-above">Use same error line as defined on error annotation line above</a></h4> |
| <p>Use the <code>//~|</code> idiom to define the same error line as the error annotation |
| line above:</p> |
| <pre><code class="language-rust ignore">struct Binder(i32, i32, i32); |
| |
| fn main() { |
| let x = Binder(1, 2, 3); |
| match x { |
| Binder(_a, _x @ ..) => {} // <- the error is on this line |
| _ => {} |
| } |
| } |
| //~^^^^ ERROR `_x @` is not allowed in a tuple struct |
| //~| ERROR this pattern has 1 field, but the corresponding tuple struct has 3 fields [E0023]</code></pre> |
| <h4 id="positioned-above-error-line"><a class="header" href="#positioned-above-error-line">Positioned above error line</a></h4> |
| <p>Use the <code>//~v</code> idiom with number of v's in the string to indicate the number |
| of lines below. This is typically used in lexer or parser tests matching on errors like unclosed |
| delimiter or unclosed literal happening at the end of file.</p> |
| <pre><code class="language-rust ignore">// ignore-tidy-trailing-newlines |
| //~v ERROR this file contains an unclosed delimiter |
| fn main((ؼ</code></pre> |
| <h4 id="error-without-line-information"><a class="header" href="#error-without-line-information">Error without line information</a></h4> |
| <p>Use <code>//~?</code> to match an error without line information. |
| <code>//~?</code> is precise and will not match errors if their line information is available. |
| It should be preferred to using <code>error-pattern</code>, which is imprecise and non-exhaustive.</p> |
| <pre><code class="language-rust ignore">//@ compile-flags: --print yyyy |
| |
| //~? ERROR unknown print request: `yyyy`</code></pre> |
| <h3 id="error-pattern"><a class="header" href="#error-pattern"><code>error-pattern</code></a></h3> |
| <p>The <code>error-pattern</code> <a href="directives.html">directive</a> can be used for runtime messages, which don't |
| have a specific span, or in exceptional cases, for compile time messages.</p> |
| <p>Let's think about this test:</p> |
| <pre><code class="language-rust ignore">fn main() { |
| let a: *const [_] = &[1, 2, 3]; |
| unsafe { |
| let _b = (*a)[3]; |
| } |
| }</code></pre> |
| <p>We want to ensure this shows "index out of bounds", but we cannot use the <code>ERROR</code> |
| annotation since the runtime error doesn't have any span. Then it's time to use the |
| <code>error-pattern</code> directive:</p> |
| <pre><code class="language-rust ignore">//@ error-pattern: index out of bounds |
| fn main() { |
| let a: *const [_] = &[1, 2, 3]; |
| unsafe { |
| let _b = (*a)[3]; |
| } |
| }</code></pre> |
| <p>Use of <code>error-pattern</code> is not recommended in general.</p> |
| <p>For strict testing of compile time output, try to use the line annotations <code>//~</code> as much as |
| possible, including <code>//~?</code> annotations for diagnostics without spans.</p> |
| <p>If the compile time output is target dependent or too verbose, use directive |
| <code>//@ dont-require-annotations: <diagnostic-kind></code> to make the line annotation checking |
| non-exhaustive. |
| Some of the compiler messages can stay uncovered by annotations in this mode.</p> |
| <p>For checking runtime output, <code>//@ check-run-results</code> may be preferable.</p> |
| <p>Only use <code>error-pattern</code> if none of the above works.</p> |
| <p>Line annotations <code>//~</code> and <code>error-pattern</code> are compatible and can be used in the same test.</p> |
| <h3 id="diagnostic-kinds-error-levels"><a class="header" href="#diagnostic-kinds-error-levels">Diagnostic kinds (error levels)</a></h3> |
| <p>The diagnostic kinds that you can have are:</p> |
| <ul> |
| <li><code>ERROR</code></li> |
| <li><code>WARN</code> (or <code>WARNING</code>)</li> |
| <li><code>NOTE</code></li> |
| <li><code>HELP</code></li> |
| <li><code>SUGGESTION</code></li> |
| <li><code>RAW</code></li> |
| </ul> |
| <p>The <code>SUGGESTION</code> kind is used for specifying what the expected replacement text |
| should be for a diagnostic suggestion. |
| The <code>RAW</code> kind can be used for matching on lines from non-structured output sometimes emitted |
| by the compiler instead of or in addition to structured json.</p> |
| <p><code>ERROR</code> and <code>WARN</code> kinds are required to be exhaustively covered by line annotations |
| <code>//~</code> by default.</p> |
| <p>Other kinds only need to be line-annotated if at least one annotation of that kind appears |
| in the test file. For example, one <code>//~ NOTE</code> will also require all other <code>//~ NOTE</code>s in the file |
| to be written out explicitly.</p> |
| <p>Use directive <code>//@ dont-require-annotations</code> to opt out of exhaustive annotations. |
| E.g. use <code>//@ dont-require-annotations: NOTE</code> to annotate notes selectively. |
| Avoid using this directive for <code>ERROR</code>s and <code>WARN</code>ings, unless there's a serious reason, like |
| target-dependent compiler output.</p> |
| <p>Some diagnostics are never required to be line-annotated, regardless of their kind or directives, |
| for example secondary lines of multiline diagnostics, |
| or ubiquitous diagnostics like <code>aborting due to N previous errors</code>.</p> |
| <p>UI tests use the <code>-A unused</code> flag by default to ignore all unused warnings, as |
| unused warnings are usually not the focus of a test. However, simple code |
| samples often have unused warnings. If the test is specifically testing an |
| unused warning, just add the appropriate <code>#![warn(unused)]</code> attribute as needed.</p> |
| <h3 id="cfg-revisions"><a class="header" href="#cfg-revisions"><code>cfg</code> revisions</a></h3> |
| <p>When using <a href="compiletest.html#revisions">revisions</a>, different messages can be |
| conditionally checked based on the current revision. This is done by placing the |
| revision cfg name in brackets like this:</p> |
| <pre><code class="language-rust ignore">//@ edition:2018 |
| //@ revisions: mir thir |
| //@[thir] compile-flags: -Z thir-unsafeck |
| |
| async unsafe fn f() {} |
| |
| async fn g() { |
| f(); //~ ERROR call to unsafe function is unsafe |
| } |
| |
| fn main() { |
| f(); //[mir]~ ERROR call to unsafe function is unsafe |
| }</code></pre> |
| <p>In this example, the second error message is only emitted in the <code>mir</code> revision. |
| The <code>thir</code> revision only emits the first error.</p> |
| <p>If the <code>cfg</code> causes the compiler to emit different output, then a test can have |
| multiple <code>.stderr</code> files for the different outputs. In the example above, there |
| would be a <code>.mir.stderr</code> and <code>.thir.stderr</code> file with the different outputs of |
| the different revisions.</p> |
| <blockquote> |
| <p>Note: cfg revisions also work inside the source code with <code>#[cfg]</code> attributes.</p> |
| <p>By convention, the <code>FALSE</code> cfg is used to have an always-false config.</p> |
| </blockquote> |
| <h2 id="controlling-passfail-expectations"><a class="header" href="#controlling-passfail-expectations">Controlling pass/fail expectations</a></h2> |
| <p>By default, a UI test is expected to <strong>generate a compile error</strong> because most |
| of the tests are checking for invalid input and error diagnostics. However, you |
| can also make UI tests where compilation is expected to succeed, and you can |
| even run the resulting program. Just add one of the following |
| <a href="directives.html">directives</a>:</p> |
| <ul> |
| <li>Pass directives: |
| <ul> |
| <li><code>//@ check-pass</code> — compilation should succeed but skip codegen |
| (which is expensive and isn't supposed to fail in most cases).</li> |
| <li><code>//@ build-pass</code> — compilation and linking should succeed but do |
| not run the resulting binary.</li> |
| <li><code>//@ run-pass</code> — compilation should succeed and running the resulting |
| binary should also succeed.</li> |
| </ul> |
| </li> |
| <li>Fail directives: |
| <ul> |
| <li><code>//@ check-fail</code> — compilation should fail (the codegen phase is skipped). |
| This is the default for UI tests.</li> |
| <li><code>//@ build-fail</code> — compilation should fail during the codegen phase. |
| This will run <code>rustc</code> twice: |
| <ul> |
| <li>First time is to ensure that the compile succeeds without the codegen phase</li> |
| <li>Second time is to ensure that the full compile fails</li> |
| </ul> |
| </li> |
| <li><code>//@ run-fail</code> — compilation should succeed, but running the resulting |
| binary should fail.</li> |
| </ul> |
| </li> |
| </ul> |
| <p>For <code>run-pass</code> and <code>run-fail</code> tests, by default the output of the program itself |
| is not checked.</p> |
| <p>If you want to check the output of running the program, include the |
| <code>check-run-results</code> directive. This will check for a <code>.run.stderr</code> and |
| <code>.run.stdout</code> files to compare against the actual output of the program.</p> |
| <p>Tests with the <code>*-pass</code> directives can be overridden with the <code>--pass</code> |
| command-line option:</p> |
| <pre><code class="language-sh">./x test tests/ui --pass check |
| </code></pre> |
| <p>The <code>--pass</code> option only affects UI tests. Using <code>--pass check</code> can run the UI |
| test suite much faster (roughly twice as fast on my system), though obviously |
| not exercising as much.</p> |
| <p>The <code>ignore-pass</code> directive can be used to ignore the <code>--pass</code> CLI flag if the |
| test won't work properly with that override.</p> |
| <h2 id="known-bugs"><a class="header" href="#known-bugs">Known bugs</a></h2> |
| <p>The <code>known-bug</code> directive may be used for tests that demonstrate a known bug |
| that has not yet been fixed. Adding tests for known bugs is helpful for several |
| reasons, including:</p> |
| <ol> |
| <li>Maintaining a functional test that can be conveniently reused when the bug is |
| fixed.</li> |
| <li>Providing a sentinel that will fail if the bug is incidentally fixed. This |
| can alert the developer so they know that the associated issue has been fixed |
| and can possibly be closed.</li> |
| </ol> |
| <p>This directive takes comma-separated issue numbers as arguments, or <code>"unknown"</code>:</p> |
| <ul> |
| <li><code>//@ known-bug: #123, #456</code> (when the issues are on rust-lang/rust)</li> |
| <li><code>//@ known-bug: rust-lang/chalk#123456</code> |
| (allows arbitrary text before the <code>#</code>, which is useful when the issue is on another repo)</li> |
| <li><code>//@ known-bug: unknown</code> |
| (when there is no known issue yet; preferably open one if it does not already exist)</li> |
| </ul> |
| <p>Do not include <a href="#error-annotations">error annotations</a> in a test with |
| <code>known-bug</code>. The test should still include other normal directives and |
| stdout/stderr files.</p> |
| <h2 id="test-organization"><a class="header" href="#test-organization">Test organization</a></h2> |
| <p>When deciding where to place a test file, please try to find a subdirectory that |
| best matches what you are trying to exercise. Do your best to keep things |
| organized. Admittedly it can be difficult as some tests can overlap different |
| categories, and the existing layout may not fit well.</p> |
| <p>Name the test by a concise description of what the test is checking. Avoid |
| including the issue number in the test name. See <a href="best-practices.html">best |
| practices</a> for a more in-depth discussion of this.</p> |
| <p>Ideally, the test should be added to a directory that helps identify what piece |
| of code is being tested here (e.g., |
| <code>tests/ui/borrowck/reject-move-out-of-borrow-via-pat.rs</code>)</p> |
| <p>When writing a new feature, you may want to <strong>create a subdirectory to store |
| your tests</strong>. For example, if you are implementing RFC 1234 ("Widgets"), then it |
| might make sense to put the tests in a directory like |
| <code>tests/ui/rfc1234-widgets/</code>.</p> |
| <p>In other cases, there may already be a suitable directory.</p> |
| <p>Over time, the <a href="https://github.com/rust-lang/rust/blob/master/tests/ui"><code>tests/ui</code></a> directory has grown very fast. There is a check in |
| <a href="intro.html#tidy">tidy</a> that will ensure none of the subdirectories has more than |
| 1000 entries. Having too many files causes problems because it isn't editor/IDE |
| friendly and the GitHub UI won't show more than 1000 entries. However, since |
| <code>tests/ui</code> (UI test root directory) and <code>tests/ui/issues</code> directories have more |
| than 1000 entries, we set a different limit for those directories. So, please |
| avoid putting a new test there and try to find a more relevant place.</p> |
| <p>For example, if your test is related to closures, you should put it in |
| <code>tests/ui/closures</code>. When you reach the limit, you could increase it by tweaking |
| <a href="https://github.com/rust-lang/rust/blob/master/src/tools/tidy/src/ui_tests.rs">here</a>.</p> |
| <h2 id="rustfix-tests"><a class="header" href="#rustfix-tests">Rustfix tests</a></h2> |
| <p>UI tests can validate that diagnostic suggestions apply correctly and that the |
| resulting changes compile correctly. This can be done with the <code>run-rustfix</code> |
| directive:</p> |
| <pre><code class="language-rust ignore">//@ run-rustfix |
| //@ check-pass |
| #![crate_type = "lib"] |
| |
| pub struct not_camel_case {} |
| //~^ WARN `not_camel_case` should have an upper camel case name |
| //~| HELP convert the identifier to upper camel case |
| //~| SUGGESTION NotCamelCase</code></pre> |
| <p>Rustfix tests should have a file with the <code>.fixed</code> extension which contains the |
| source file after the suggestion has been applied.</p> |
| <ul> |
| <li>When the test is run, compiletest first checks that the correct lint/warning |
| is generated.</li> |
| <li>Then, it applies the suggestion and compares against <code>.fixed</code> (they must |
| match).</li> |
| <li>Finally, the fixed source is compiled, and this compilation is required to |
| succeed.</li> |
| </ul> |
| <p>Usually when creating a rustfix test you will generate the <code>.fixed</code> file |
| automatically with the <code>x test --bless</code> option.</p> |
| <p>The <code>run-rustfix</code> directive will cause <em>all</em> suggestions to be applied, even if |
| they are not <a href="../diagnostics.html#suggestions"><code>MachineApplicable</code></a>. If this is a |
| problem, then you can add the <code>rustfix-only-machine-applicable</code> directive in |
| addition to <code>run-rustfix</code>. This should be used if there is a mixture of |
| different suggestion levels, and some of the non-machine-applicable ones do not |
| apply cleanly.</p> |
| <h2 id="compare-modes"><a class="header" href="#compare-modes">Compare modes</a></h2> |
| <p><a href="compiletest.html#compare-modes">Compare modes</a> can be used to run all tests with |
| different flags from what they are normally compiled with. In some cases, this |
| might result in different output from the compiler. To support this, different |
| output files can be saved which contain the output based on the compare mode.</p> |
| <p>For example, when using the Polonius mode, a test <code>foo.rs</code> will first look for |
| expected output in <code>foo.polonius.stderr</code>, falling back to the usual <code>foo.stderr</code> |
| if not found. This is useful as different modes can sometimes result in |
| different diagnostics and behavior. This can help track which tests have |
| differences between the modes, and to visually inspect those diagnostic |
| differences.</p> |
| <p>If in the rare case you encounter a test that has different behavior, you can |
| run something like the following to generate the alternate stderr file:</p> |
| <pre><code class="language-sh">./x test tests/ui --compare-mode=polonius --bless |
| </code></pre> |
| <p>Currently none of the compare modes are checked in CI for UI tests.</p> |
| <h2 id="rustc_-test-attributes"><a class="header" href="#rustc_-test-attributes"><code>rustc_*</code> TEST attributes</a></h2> |
| <p>The compiler defines several perma-unstable <code>#[rustc_*]</code> attributes gated behind |
| the internal feature <code>rustc_attrs</code> that dump extra compiler-internal |
| information. See the corresponding subsection in <a href="../compiler-debugging.html#rustc_-test-attributes">compiler debugging</a> for more |
| details.</p> |
| <p>They can be used in tests to more precisely, legibly and easily test internal |
| compiler state in cases where it would otherwise be very hard to do the same |
| with "user-facing" Rust alone. Indeed, one could say that this slightly abuses |
| the term "UI" (<em>user</em> interface) and turns such UI tests from black-box tests |
| into white-box ones. Use them carefully and sparingly.</p> |
| <h2 id="ui-test-mode-preset-lint-levels"><a class="header" href="#ui-test-mode-preset-lint-levels">UI test mode preset lint levels</a></h2> |
| <p>By default, test suites under UI test mode (<code>tests/ui</code>, <code>tests/ui-fulldeps</code>, |
| but not <code>tests/rustdoc-ui</code>) will specify</p> |
| <ul> |
| <li><code>-A unused</code></li> |
| <li><code>-A internal_features</code></li> |
| </ul> |
| <p>If:</p> |
| <ul> |
| <li>The ui test's pass mode is below <code>run</code> (i.e. check or build).</li> |
| <li>No compare modes are specified.</li> |
| </ul> |
| <p>Since they can be very noisy in ui tests.</p> |
| <p>You can override them with <code>compile-flags</code> lint level flags or |
| in-source lint level attributes as required.</p> |
| <p>Note that the <code>rustfix</code> version will <em>not</em> have <code>-A unused</code> passed, |
| meaning that you may have to <code>#[allow(unused)]</code> to suppress <code>unused</code> |
| lints on the rustfix'd file (because we might be testing rustfix |
| on <code>unused</code> lints themselves).</p> |
| <hr> |
| <ol class="footnote-definition"><li id="footnote-main test file"> |
| <p>This is a file that has the <code>~?</code> annotations, |
| as distinct from aux files, or sources that we have no control over. <a href="#fr-main test file-1">↩</a></p> |
| </li> |
| </ol> |
| </main> |
| |
| <nav class="nav-wrapper" aria-label="Page navigation"> |
| <!-- Mobile navigation buttons --> |
| <a rel="prev" href="../tests/compiletest.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> |
| <i class="fa fa-angle-left"></i> |
| </a> |
| |
| <a rel="next prefetch" href="../tests/directives.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> |
| <i class="fa fa-angle-right"></i> |
| </a> |
| |
| <div style="clear: both"></div> |
| </nav> |
| </div> |
| </div> |
| |
| <nav class="nav-wide-wrapper" aria-label="Page navigation"> |
| <a rel="prev" href="../tests/compiletest.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left"> |
| <i class="fa fa-angle-left"></i> |
| </a> |
| |
| <a rel="next prefetch" href="../tests/directives.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right"> |
| <i class="fa fa-angle-right"></i> |
| </a> |
| </nav> |
| |
| </div> |
| |
| |
| |
| |
| <script> |
| window.playground_copyable = true; |
| </script> |
| |
| |
| <script src="../elasticlunr.min.js"></script> |
| <script src="../mark.min.js"></script> |
| <script src="../searcher.js"></script> |
| |
| <script src="../clipboard.min.js"></script> |
| <script src="../highlight.js"></script> |
| <script src="../book.js"></script> |
| |
| <!-- Custom JS scripts --> |
| <script src="../mermaid.min.js"></script> |
| <script src="../mermaid-init.js"></script> |
| |
| |
| </div> |
| </body> |
| </html> |
