blob: 9e26e2e468140aee0a0cc78a696289e45f64c342 [file] [log] [blame] [edit]
<!DOCTYPE HTML>
<html lang="en" class="light sidebar-visible" dir="ltr">
<head>
<!-- Book generated using mdBook -->
<meta charset="UTF-8">
<title>Writing Tests - Cargo Contributor Guide</title>
<!-- Custom HTML head -->
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff">
<link rel="icon" href="../favicon.svg">
<link rel="shortcut icon" href="../favicon.png">
<link rel="stylesheet" href="../css/variables.css">
<link rel="stylesheet" href="../css/general.css">
<link rel="stylesheet" href="../css/chrome.css">
<link rel="stylesheet" href="../css/print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="../FontAwesome/css/font-awesome.css">
<link rel="stylesheet" href="../fonts/fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" id="highlight-css" href="../highlight.css">
<link rel="stylesheet" id="tomorrow-night-css" href="../tomorrow-night.css">
<link rel="stylesheet" id="ayu-highlight-css" href="../ayu-highlight.css">
<!-- Custom theme stylesheets -->
<!-- Provide site root and default themes to javascript -->
<script>
const path_to_root = "../";
const default_light_theme = "light";
const default_dark_theme = "navy";
window.path_to_searchindex_js = "../searchindex.js";
</script>
<!-- Start loading toc.js asap -->
<script src="../toc.js"></script>
</head>
<body>
<div id="mdbook-help-container">
<div id="mdbook-help-popup">
<h2 class="mdbook-help-title">Keyboard shortcuts</h2>
<div>
<p>Press <kbd></kbd> or <kbd></kbd> to navigate between chapters</p>
<p>Press <kbd>S</kbd> or <kbd>/</kbd> to search in the book</p>
<p>Press <kbd>?</kbd> to show this help</p>
<p>Press <kbd>Esc</kbd> to hide this help</p>
</div>
</div>
</div>
<div id="body-container">
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script>
try {
let theme = localStorage.getItem('mdbook-theme');
let sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script>
const default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? default_dark_theme : default_light_theme;
let theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
const html = document.documentElement;
html.classList.remove('light')
html.classList.add(theme);
html.classList.add("js");
</script>
<input type="checkbox" id="sidebar-toggle-anchor" class="hidden">
<!-- Hide / unhide sidebar before it is displayed -->
<script>
let sidebar = null;
const sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
} else {
sidebar = 'hidden';
sidebar_toggle.checked = false;
}
if (sidebar === 'visible') {
sidebar_toggle.checked = true;
} else {
html.classList.remove('sidebar-visible');
}
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<!-- populated by js -->
<mdbook-sidebar-scrollbox class="sidebar-scrollbox"></mdbook-sidebar-scrollbox>
<noscript>
<iframe class="sidebar-iframe-outer" src="../toc.html"></iframe>
</noscript>
<div id="sidebar-resize-handle" class="sidebar-resize-handle">
<div class="sidebar-resize-indicator"></div>
</div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky">
<div class="left-buttons">
<label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</label>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="default_theme">Auto</button></li>
<li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
<button id="search-toggle" class="icon-button" type="button" title="Search (`/`)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="/ s" aria-controls="searchbar">
<i class="fa fa-search"></i>
</button>
</div>
<h1 class="menu-title">Cargo Contributor Guide</h1>
<div class="right-buttons">
<a href="../print.html" title="Print this book" aria-label="Print this book">
<i id="print-button" class="fa fa-print"></i>
</a>
<a href="https://github.com/rust-lang/cargo/tree/master/src/doc/contrib/src" title="Git repository" aria-label="Git repository">
<i id="git-repository-button" class="fa fa-github"></i>
</a>
<a href="https://github.com/rust-lang/cargo/edit/master/src/doc/contrib/src/tests/writing.md" title="Suggest an edit" aria-label="Suggest an edit" rel="edit">
<i id="git-edit-button" class="fa fa-edit"></i>
</a>
</div>
</div>
<div id="search-wrapper" class="hidden">
<form id="searchbar-outer" class="searchbar-outer">
<div class="search-wrapper">
<input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
<div class="spinner-wrapper">
<i class="fa fa-spinner fa-spin"></i>
</div>
</div>
</form>
<div id="searchresults-outer" class="searchresults-outer hidden">
<div id="searchresults-header" class="searchresults-header"></div>
<ul id="searchresults">
</ul>
</div>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script>
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1 id="writing-tests"><a class="header" href="#writing-tests">Writing Tests</a></h1>
<p>The following focuses on writing an integration test. However, writing unit
tests is also encouraged!</p>
<h2 id="testsuite"><a class="header" href="#testsuite">Testsuite</a></h2>
<p>Cargo has a wide variety of integration tests that execute the <code>cargo</code> binary
and verify its behavior, located in the <a href="https://github.com/rust-lang/cargo/tree/master/tests/testsuite/"><code>testsuite</code></a> directory. The
<a href="https://github.com/rust-lang/cargo/blob/master/crates/cargo-test-support/src/lib.rs"><code>support</code></a> crate and <a href="https://docs.rs/snapbox/latest/snapbox/"><code>snapbox</code></a> contain many helpers to make this process easy.</p>
<p>There are two styles of tests that can roughly be categorized as</p>
<ul>
<li>functional tests
<ul>
<li>The fixture is programmatically defined</li>
<li>The assertions may be in-source snapshots, hard-coded strings, or programmatically generated</li>
<li>Easier to share in an issue as a code block is completely self-contained</li>
</ul>
</li>
<li>ui tests
<ul>
<li>The fixture is file-based</li>
<li>The assertions use file-backed snapshots that can be updated with an env variable</li>
<li>Easier to review the expected behavior of the command as more details are included</li>
<li>Easier to get up and running from an existing project</li>
<li>Easier to reason about as everything is just files in the repo</li>
</ul>
</li>
</ul>
<p>These tests typically work by creating a temporary “project” with a
<code>Cargo.toml</code> file, executing the <code>cargo</code> binary process, and checking the
stdout and stderr output against the expected output.</p>
<h3 id="functional-tests"><a class="header" href="#functional-tests">Functional Tests</a></h3>
<p>Generally, a functional test will be placed in <code>tests/testsuite/&lt;command&gt;.rs</code> and will look roughly like:</p>
<pre><code class="language-rust ignore">use crate::prelude::*;
use cargo_test_support::str;
use cargo_test_support::project;
#[cargo_test]
fn &lt;description&gt;() {
let p = project()
.file("src/main.rs", r#"fn main() { println!("hi!"); }"#)
.build();
p.cargo("run --bin foo")
.with_stderr_data(str![[r#"
[COMPILING] foo [..]
[FINISHED] [..]
[RUNNING] `target/debug/foo`
"#]])
.with_stdout_data(str![["hi!"]])
.run();
}</code></pre>
<p>The <a href="https://doc.rust-lang.org/nightly/nightly-rustc/cargo_test_macro/attr.cargo_test.html"><code>#[cargo_test]</code> attribute</a> is used in place of
<code>#[test]</code> to inject some setup code and declare requirements for running the
test.</p>
<p><a href="https://github.com/rust-lang/cargo/blob/d847468768446168b596f721844193afaaf9d3f2/crates/cargo-test-support/src/lib.rs#L196-L202"><code>ProjectBuilder</code></a> via <code>project()</code>:</p>
<ul>
<li>Each project is in a separate directory in the sandbox</li>
<li>If you do not specify a <code>Cargo.toml</code> manifest using <code>file()</code>, one is
automatically created with a project name of <code>foo</code> using <code>basic_manifest()</code>.</li>
</ul>
<p><a href="https://github.com/rust-lang/cargo/blob/d847468768446168b596f721844193afaaf9d3f2/crates/cargo-test-support/src/lib.rs#L531-L550"><code>Execs</code></a> via <code>p.cargo(...)</code>:</p>
<ul>
<li>This executes the command and evaluates different assertions
<ul>
<li>See <a href="https://github.com/rust-lang/cargo/blob/master/crates/cargo-test-support/src/compare.rs"><code>support::compare</code></a> for an explanation of the string pattern matching.
Patterns are used to make it easier to match against the expected output.</li>
</ul>
</li>
</ul>
<h4 id="filesystem-layout-testing"><a class="header" href="#filesystem-layout-testing">Filesystem layout testing</a></h4>
<p>Tests often to need to verify Cargo created/removed files.
The <code>CargoPathExt</code> trait (implemented by <code>Path</code> and <code>PathBuf</code>) provides a <code>assert_dir_layout()</code> to verify the files in a directory (including nested directories).
This takes a snapshot of file paths for the given directory and asserts that all files are present and no new files have been created.
This function also takes a list of patterns to ignore from the snapshot to make working with platform specific files easier.</p>
<p>Note: You will commonly need to call <code>unordered()</code> before passing your snapshot to deal with platform differences like binaries having <code>.exe</code> on Windows.
<code>assert_build_dir_layout</code> is a more specialized version of <code>assert_dir_layout()</code> that is automatically unordered and ignores common platform specific files designed for the Cargo build cache.</p>
<h4 id="testing-nightly-features"><a class="header" href="#testing-nightly-features">Testing Nightly Features</a></h4>
<p>If you are testing a Cargo feature that only works on “nightly” Cargo, then
you need to call <code>masquerade_as_nightly_cargo</code> on the process builder and pass
the name of the feature as the reason, like this:</p>
<pre><code class="language-rust ignore">p.cargo("build").masquerade_as_nightly_cargo(&amp;["print-im-a-teapot"])</code></pre>
<p>If you are testing a feature that only works on <em>nightly rustc</em> (such as
benchmarks), then you should use the <code>nightly</code> option of the <code>cargo_test</code>
attribute, like this:</p>
<pre><code class="language-rust ignore">#[cargo_test(nightly, reason = "-Zfoo is unstable")]</code></pre>
<p>This will cause the test to be ignored if not running on the nightly toolchain.</p>
<h4 id="specifying-dependencies"><a class="header" href="#specifying-dependencies">Specifying Dependencies</a></h4>
<p>You should not write any tests that use the network such as contacting
crates.io. Typically, simple path dependencies are the easiest way to add a
dependency. Example:</p>
<pre><code class="language-rust ignore">let p = project()
.file("Cargo.toml", r#"
[package]
name = "foo"
version = "1.0.0"
[dependencies]
bar = {path = "bar"}
"#)
.file("src/lib.rs", "extern crate bar;")
.file("bar/Cargo.toml", &amp;basic_manifest("bar", "1.0.0"))
.file("bar/src/lib.rs", "")
.build();</code></pre>
<p>If you need to test with registry dependencies, see
<a href="https://github.com/rust-lang/cargo/blob/d847468768446168b596f721844193afaaf9d3f2/crates/cargo-test-support/src/registry.rs#L311-L389"><code>support::registry::Package</code></a> for creating packages you can depend on.</p>
<p>If you need to test git dependencies, see <a href="https://github.com/rust-lang/cargo/blob/master/crates/cargo-test-support/src/git.rs"><code>support::git</code></a> to create a git
dependency.</p>
<h4 id="cross-compilation"><a class="header" href="#cross-compilation">Cross compilation</a></h4>
<p>There are some utilities to help support tests that need to work against a
target other than the host. See <a href="running.html#running-cross-tests">Running cross
tests</a> for more an introduction on cross
compilation tests.</p>
<p>Tests that need to do cross-compilation should include this at the top of the
test to disable it in scenarios where cross compilation isn’t available:</p>
<pre><code class="language-rust ignore">if crate::utils::cross_compile::disabled() {
return;
}</code></pre>
<p>The name of the target can be fetched with the <a href="https://github.com/rust-lang/cargo/blob/d58902e22e148426193cf3b8c4449fd3c05c0afd/crates/cargo-test-support/src/cross_compile.rs#L208-L225"><code>cross_compile::alternate()</code></a>
function. The name of the host target can be fetched with
<a href="https://github.com/rust-lang/cargo/blob/d58902e22e148426193cf3b8c4449fd3c05c0afd/crates/cargo-test-support/src/lib.rs#L1137-L1140"><code>cargo_test_support::rustc_host()</code></a>.</p>
<p>If the test needs to run the cross-compiled binary, then it should have
something like this to exit the test before doing so:</p>
<pre><code class="language-rust ignore">if crate::utils::cross_compile::can_run_on_host() {
return;
}</code></pre>
<h3 id="ui-tests"><a class="header" href="#ui-tests">UI Tests</a></h3>
<p>UI Tests are a bit more spread out and generally look like:</p>
<p><code>tests/testsuite/&lt;command&gt;/mod.rs</code>:</p>
<pre><code class="language-rust ignore">mod &lt;case&gt;;</code></pre>
<p><code>tests/testsuite/&lt;command&gt;/&lt;case&gt;/mod.rs</code>:</p>
<pre><code class="language-rust ignore">use crate::prelude::*;
use cargo_test_support::compare::assert_ui;
use cargo_test_support::current_dir;
use cargo_test_support::file;
use cargo_test_support::Project;
#[cargo_test]
fn case() {
let project = Project::from_template(current_dir!().join("in"));
let project_root = project.root();
let cwd = &amp;project_root;
snapbox::cmd::Command::cargo_ui()
.arg("run")
.arg_line("--bin foo")
.current_dir(cwd)
.assert()
.success()
.stdout_matches(file!("stdout.log"))
.stderr_matches(file!("stderr.log"));
assert_ui().subset_matches(current_dir!().join("out"), &amp;project_root);
}</code></pre>
<p>Then populate</p>
<ul>
<li><code>tests/testsuite/&lt;command&gt;/&lt;case&gt;/in</code> with the project’s directory structure</li>
<li><code>tests/testsuite/&lt;command&gt;/&lt;case&gt;/out</code> with the files you want verified</li>
<li><code>tests/testsuite/&lt;command&gt;/&lt;case&gt;/stdout.log</code> with nothing</li>
<li><code>tests/testsuite/&lt;command&gt;/&lt;case&gt;/stderr.log</code> with nothing</li>
</ul>
<p><code>#[cargo_test]</code>:</p>
<ul>
<li>This is used in place of <code>#[test]</code></li>
<li>This attribute injects code which does some setup before starting the
test, creating a filesystem “sandbox” under the “cargo integration test”
directory for each test such as
<code>/path/to/cargo/target/cit/t123/</code></li>
<li>The sandbox will contain a <code>home</code> directory that will be used instead of your normal home directory</li>
</ul>
<p><code>Project</code>:</p>
<ul>
<li>The project is copied from a directory in the repo</li>
<li>Each project is in a separate directory in the sandbox</li>
</ul>
<p><a href="https://docs.rs/snapbox/latest/snapbox/cmd/struct.Command.html"><code>Command</code></a> via <code>Command::cargo_ui()</code>:</p>
<ul>
<li>Set up and run a command.</li>
</ul>
<p><a href="https://docs.rs/snapbox/latest/snapbox/cmd/struct.OutputAssert.html"><code>OutputAssert</code></a> via <code>Command::assert()</code>:</p>
<ul>
<li>Perform assertions on the result of the <a href="https://docs.rs/snapbox/latest/snapbox/cmd/struct.Command.html"><code>Command</code></a></li>
</ul>
<p><a href="https://docs.rs/snapbox/latest/snapbox/struct.Assert.html"><code>Assert</code></a> via <code>assert_ui()</code>:</p>
<ul>
<li>Verify the command modified the file system as expected</li>
</ul>
<h4 id="updating-snapshots"><a class="header" href="#updating-snapshots">Updating Snapshots</a></h4>
<p>The project, stdout, and stderr snapshots can be updated by running with the
<code>SNAPSHOTS=overwrite</code> environment variable, like:</p>
<pre><code class="language-console">$ SNAPSHOTS=overwrite cargo test
</code></pre>
<p>Be sure to check the snapshots to make sure they make sense.</p>
<h4 id="testing-nightly-features-1"><a class="header" href="#testing-nightly-features-1">Testing Nightly Features</a></h4>
<p>If you are testing a Cargo feature that only works on “nightly” Cargo, then
you need to call <code>masquerade_as_nightly_cargo</code> on the process builder and pass
the name of the feature as the reason, like this:</p>
<pre><code class="language-rust ignore"> snapbox::cmd::Command::cargo()
.masquerade_as_nightly_cargo(&amp;["print-im-a-teapot"])</code></pre>
<p>If you are testing a feature that only works on <em>nightly rustc</em> (such as
benchmarks), then you should use the <code>nightly</code> option of the <code>cargo_test</code>
attribute, like this:</p>
<pre><code class="language-rust ignore">#[cargo_test(nightly, reason = "-Zfoo is unstable")]</code></pre>
<p>This will cause the test to be ignored if not running on the nightly toolchain.</p>
<h3 id="platform-specific-notes"><a class="header" href="#platform-specific-notes">Platform-specific Notes</a></h3>
<p>When checking output, use <code>/</code> for paths even on Windows: the actual output
of <code>\</code> on Windows will be replaced with <code>/</code>.</p>
<p>Be careful when executing binaries on Windows. You should not rename, delete,
or overwrite a binary immediately after running it. Under some conditions
Windows will fail with errors like “directory not empty” or “failed to remove”
or “access is denied”.</p>
<h2 id="debugging-tests"><a class="header" href="#debugging-tests">Debugging tests</a></h2>
<p>In some cases, you may need to dig into a test that is not working as you
expect, or you just generally want to experiment within the sandbox
environment. The general process is:</p>
<ol>
<li>
<p>Build the sandbox for the test you want to investigate. For example:</p>
<p><code>cargo test --test testsuite -- features2::inactivate_targets</code>.</p>
</li>
<li>
<p>In another terminal, head into the sandbox directory to inspect the files and run <code>cargo</code> directly.</p>
<ol>
<li>
<p>The sandbox directories start with <code>t0</code> for the first test.</p>
<p><code>cd target/tmp/cit/t0</code></p>
</li>
<li>
<p>Set up the environment so that the sandbox configuration takes effect:</p>
<p><code>export CARGO_HOME=$(pwd)/home/.cargo</code></p>
</li>
<li>
<p>Most tests create a <code>foo</code> project, so head into that:</p>
<p><code>cd foo</code></p>
</li>
</ol>
</li>
<li>
<p>Run whatever cargo command you want. See <a href="../process/working-on-cargo.html#running-cargo">Running Cargo</a> for more details
on running the correct <code>cargo</code> process. Some examples:</p>
<ul>
<li><code>/path/to/my/cargo/target/debug/cargo check</code></li>
<li>Using a debugger like <code>lldb</code> or <code>gdb</code>:
<ol>
<li><code>lldb /path/to/my/cargo/target/debug/cargo</code></li>
<li>Set a breakpoint, for example: <code>b generate_root_units</code></li>
<li>Run with arguments: <code>r check</code></li>
</ol>
</li>
</ul>
</li>
</ol>
</main>
<nav class="nav-wrapper" aria-label="Page navigation">
<!-- Mobile navigation buttons -->
<a rel="prev" href="../tests/running.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next prefetch" href="../tests/profiling.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
<div style="clear: both"></div>
</nav>
</div>
</div>
<nav class="nav-wide-wrapper" aria-label="Page navigation">
<a rel="prev" href="../tests/running.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next prefetch" href="../tests/profiling.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
</nav>
</div>
<script>
window.playground_copyable = true;
</script>
<script src="../elasticlunr.min.js"></script>
<script src="../mark.min.js"></script>
<script src="../searcher.js"></script>
<script src="../clipboard.min.js"></script>
<script src="../highlight.js"></script>
<script src="../book.js"></script>
<!-- Custom JS scripts -->
</div>
</body>
</html>