| use std::net::{SocketAddr, TcpStream}; |
| use std::process::{Command, Stdio, exit}; |
| use std::time::Duration; |
| use std::{env, fs, process, thread}; |
| |
| const JOSH_PORT: u16 = 42042; |
| const DEFAULT_PR_BRANCH: &str = "update-builtins"; |
| |
| pub struct GitSync { |
| upstream_repo: String, |
| upstream_ref: String, |
| upstream_url: String, |
| josh_filter: String, |
| josh_url_base: String, |
| } |
| |
| /// This code was adapted from the miri repository, via the rustc-dev-guide |
| /// (<https://github.com/rust-lang/rustc-dev-guide/tree/c51adbd12d/josh-sync>) |
| impl GitSync { |
| pub fn from_current_dir() -> Self { |
| let upstream_repo = |
| env::var("UPSTREAM_ORG").unwrap_or_else(|_| "rust-lang".to_owned()) + "/rust"; |
| |
| Self { |
| upstream_url: format!("https://github.com/{upstream_repo}"), |
| upstream_repo, |
| upstream_ref: env::var("UPSTREAM_REF").unwrap_or_else(|_| "HEAD".to_owned()), |
| josh_filter: ":/library/compiler-builtins".to_owned(), |
| josh_url_base: format!("http://localhost:{JOSH_PORT}"), |
| } |
| } |
| |
| /// Pull from rust-lang/rust to compiler-builtins. |
| pub fn rustc_pull(&self, commit: Option<String>) { |
| let Self { |
| upstream_ref, |
| upstream_url, |
| upstream_repo, |
| .. |
| } = self; |
| |
| let new_upstream_base = commit.unwrap_or_else(|| { |
| let out = check_output(["git", "ls-remote", upstream_url, upstream_ref]); |
| out.split_whitespace() |
| .next() |
| .unwrap_or_else(|| panic!("could not split output: '{out}'")) |
| .to_owned() |
| }); |
| |
| ensure_clean(); |
| |
| // Make sure josh is running. |
| let _josh = Josh::start(); |
| let josh_url_filtered = self.josh_url( |
| &self.upstream_repo, |
| Some(&new_upstream_base), |
| Some(&self.josh_filter), |
| ); |
| |
| let previous_upstream_base = fs::read_to_string("rust-version") |
| .expect("failed to read `rust-version`") |
| .trim() |
| .to_string(); |
| assert_ne!(previous_upstream_base, new_upstream_base, "nothing to pull"); |
| |
| let orig_head = check_output(["git", "rev-parse", "HEAD"]); |
| println!("original upstream base: {previous_upstream_base}"); |
| println!("new upstream base: {new_upstream_base}"); |
| println!("original HEAD: {orig_head}"); |
| |
| // Fetch the latest upstream HEAD so we can get a summary. Use the Josh URL for caching. |
| run([ |
| "git", |
| "fetch", |
| &self.josh_url(&self.upstream_repo, Some(&new_upstream_base), Some(":/")), |
| &new_upstream_base, |
| "--depth=1", |
| ]); |
| let new_summary = check_output(["git", "log", "-1", "--format=%h %s", &new_upstream_base]); |
| |
| // Update rust-version file. As a separate commit, since making it part of |
| // the merge has confused the heck out of josh in the past. |
| // We pass `--no-verify` to avoid running git hooks. |
| // We do this before the merge so that if there are merge conflicts, we have |
| // the right rust-version file while resolving them. |
| fs::write("rust-version", format!("{new_upstream_base}\n")) |
| .expect("failed to write rust-version"); |
| |
| let prep_message = format!( |
| "Update the upstream Rust version\n\n\ |
| To prepare for merging from {upstream_repo}, set the version file to:\n\n \ |
| {new_summary}\n\ |
| ", |
| ); |
| run([ |
| "git", |
| "commit", |
| "rust-version", |
| "--no-verify", |
| "-m", |
| &prep_message, |
| ]); |
| |
| // Fetch given rustc commit. |
| run(["git", "fetch", &josh_url_filtered]); |
| let incoming_ref = check_output(["git", "rev-parse", "FETCH_HEAD"]); |
| println!("incoming ref: {incoming_ref}"); |
| |
| let merge_message = format!( |
| "Merge ref '{upstream_head_short}{filter}' from {upstream_url}\n\n\ |
| Pull recent changes from {upstream_repo} via Josh.\n\n\ |
| Upstream ref: {new_upstream_base}\n\ |
| Filtered ref: {incoming_ref}\n\ |
| ", |
| upstream_head_short = &new_upstream_base[..12], |
| filter = self.josh_filter |
| ); |
| |
| // This should not add any new root commits. So count those before and after merging. |
| let num_roots = || -> u32 { |
| let out = check_output(["git", "rev-list", "HEAD", "--max-parents=0", "--count"]); |
| out.trim() |
| .parse::<u32>() |
| .unwrap_or_else(|e| panic!("failed to parse `{out}`: {e}")) |
| }; |
| let num_roots_before = num_roots(); |
| |
| let pre_merge_sha = check_output(["git", "rev-parse", "HEAD"]); |
| println!("pre-merge HEAD: {pre_merge_sha}"); |
| |
| // Merge the fetched commit. |
| run([ |
| "git", |
| "merge", |
| "FETCH_HEAD", |
| "--no-verify", |
| "--no-ff", |
| "-m", |
| &merge_message, |
| ]); |
| |
| let current_sha = check_output(["git", "rev-parse", "HEAD"]); |
| if current_sha == pre_merge_sha { |
| run(["git", "reset", "--hard", &orig_head]); |
| eprintln!( |
| "No merge was performed, no changes to pull were found. \ |
| Rolled back the preparation commit." |
| ); |
| exit(1); |
| } |
| |
| // Check that the number of roots did not increase. |
| assert_eq!( |
| num_roots(), |
| num_roots_before, |
| "Josh created a new root commit. This is probably not the history you want." |
| ); |
| } |
| |
| /// Construct an update to rust-lang/rust from compiler-builtins. |
| pub fn rustc_push(&self, github_user: &str, branch: Option<&str>) { |
| let Self { |
| josh_filter, |
| upstream_url, |
| .. |
| } = self; |
| |
| let branch = branch.unwrap_or(DEFAULT_PR_BRANCH); |
| let josh_url = self.josh_url(&format!("{github_user}/rust"), None, Some(josh_filter)); |
| let user_upstream_url = format!("git@github.com:{github_user}/rust.git"); |
| |
| let Ok(rustc_git) = env::var("RUSTC_GIT") else { |
| panic!("the RUSTC_GIT environment variable must be set to a rust-lang/rust checkout") |
| }; |
| |
| ensure_clean(); |
| let base = fs::read_to_string("rust-version") |
| .expect("failed to read `rust-version`") |
| .trim() |
| .to_string(); |
| |
| // Make sure josh is running. |
| let _josh = Josh::start(); |
| |
| // Prepare the branch. Pushing works much better if we use as base exactly |
| // the commit that we pulled from last time, so we use the `rust-version` |
| // file to find out which commit that would be. |
| println!("Preparing {github_user}/rust (base: {base})..."); |
| |
| if Command::new("git") |
| .args(["-C", &rustc_git, "fetch", &user_upstream_url, branch]) |
| .output() // capture output |
| .expect("could not run fetch") |
| .status |
| .success() |
| { |
| panic!( |
| "The branch '{branch}' seems to already exist in '{user_upstream_url}'. \ |
| Please delete it and try again." |
| ); |
| } |
| |
| run(["git", "-C", &rustc_git, "fetch", upstream_url, &base]); |
| |
| run_cfg("git", |c| { |
| c.args([ |
| "-C", |
| &rustc_git, |
| "push", |
| &user_upstream_url, |
| &format!("{base}:refs/heads/{branch}"), |
| ]) |
| .stdout(Stdio::null()) |
| .stderr(Stdio::null()) // silence the "create GitHub PR" message |
| }); |
| println!("pushed PR branch"); |
| |
| // Do the actual push. |
| println!("Pushing changes..."); |
| run(["git", "push", &josh_url, &format!("HEAD:{branch}")]); |
| println!(); |
| |
| // Do a round-trip check to make sure the push worked as expected. |
| run(["git", "fetch", &josh_url, branch]); |
| |
| let head = check_output(["git", "rev-parse", "HEAD"]); |
| let fetch_head = check_output(["git", "rev-parse", "FETCH_HEAD"]); |
| assert_eq!( |
| head, fetch_head, |
| "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ |
| Expected {head}, got {fetch_head}." |
| ); |
| println!( |
| "Confirmed that the push round-trips back to compiler-builtins properly. Please \ |
| create a rustc PR:" |
| ); |
| // Open PR with `subtree update` title to silence the `no-merges` triagebot check |
| println!( |
| " {upstream_url}/compare/{github_user}:{branch}?quick_pull=1\ |
| &title=Update%20the%20%60compiler-builtins%60%20subtree\ |
| &body=Update%20the%20Josh%20subtree%20to%20https%3A%2F%2Fgithub.com%2Frust-lang%2F\ |
| compiler-builtins%2Fcommit%2F{head_short}.%0A%0Ar%3F%20%40ghost", |
| head_short = &head[..12], |
| ); |
| } |
| |
| /// Construct a url to the local Josh server with (optionally) |
| fn josh_url(&self, repo: &str, rev: Option<&str>, filter: Option<&str>) -> String { |
| format!( |
| "{base}/{repo}.git{at}{rev}{filter}{filt_git}", |
| base = self.josh_url_base, |
| at = if rev.is_some() { "@" } else { "" }, |
| rev = rev.unwrap_or_default(), |
| filter = filter.unwrap_or_default(), |
| filt_git = if filter.is_some() { ".git" } else { "" } |
| ) |
| } |
| } |
| |
| /// Fail if there are files that need to be checked in. |
| fn ensure_clean() { |
| let read = check_output(["git", "status", "--untracked-files=no", "--porcelain"]); |
| assert!( |
| read.is_empty(), |
| "working directory must be clean before performing rustc pull" |
| ); |
| } |
| |
| /* Helpers for running commands with logged invocations */ |
| |
| /// Run a command from an array, passing its output through. |
| fn run<'a, Args: AsRef<[&'a str]>>(l: Args) { |
| let l = l.as_ref(); |
| run_cfg(l[0], |c| c.args(&l[1..])); |
| } |
| |
| /// Run a command from an array, collecting its output. |
| fn check_output<'a, Args: AsRef<[&'a str]>>(l: Args) -> String { |
| let l = l.as_ref(); |
| check_output_cfg(l[0], |c| c.args(&l[1..])) |
| } |
| |
| /// [`run`] with configuration. |
| fn run_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) { |
| // self.read(l.as_ref()); |
| check_output_cfg(prog, |c| f(c.stdout(Stdio::inherit()))); |
| } |
| |
| /// [`read`] with configuration. All shell helpers print the command and pass stderr. |
| fn check_output_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) -> String { |
| let mut cmd = Command::new(prog); |
| cmd.stderr(Stdio::inherit()); |
| f(&mut cmd); |
| eprintln!("+ {cmd:?}"); |
| let out = cmd.output().expect("command failed"); |
| assert!(out.status.success()); |
| String::from_utf8(out.stdout.trim_ascii().to_vec()).expect("non-UTF8 output") |
| } |
| |
| /// Create a wrapper that stops Josh on drop. |
| pub struct Josh(process::Child); |
| |
| impl Josh { |
| pub fn start() -> Self { |
| // Determine cache directory. |
| let user_dirs = |
| directories::ProjectDirs::from("org", "rust-lang", "rustc-compiler-builtins-josh") |
| .unwrap(); |
| let local_dir = user_dirs.cache_dir().to_owned(); |
| |
| // Start josh, silencing its output. |
| #[expect(clippy::zombie_processes, reason = "clippy can't handle the loop")] |
| let josh = process::Command::new("josh-proxy") |
| .arg("--local") |
| .arg(local_dir) |
| .args([ |
| "--remote=https://github.com", |
| &format!("--port={JOSH_PORT}"), |
| "--no-background", |
| ]) |
| .stdout(Stdio::null()) |
| .stderr(Stdio::null()) |
| .spawn() |
| .expect("failed to start josh-proxy, make sure it is installed"); |
| |
| // Wait until the port is open. We try every 10ms until 1s passed. |
| for _ in 0..100 { |
| // This will generally fail immediately when the port is still closed. |
| let addr = SocketAddr::from(([127, 0, 0, 1], JOSH_PORT)); |
| let josh_ready = TcpStream::connect_timeout(&addr, Duration::from_millis(1)); |
| |
| if josh_ready.is_ok() { |
| println!("josh up and running"); |
| return Josh(josh); |
| } |
| |
| // Not ready yet. |
| thread::sleep(Duration::from_millis(10)); |
| } |
| panic!("Even after waiting for 1s, josh-proxy is still not available.") |
| } |
| } |
| |
| impl Drop for Josh { |
| fn drop(&mut self) { |
| if cfg!(unix) { |
| // Try to gracefully shut it down. |
| Command::new("kill") |
| .args(["-s", "INT", &self.0.id().to_string()]) |
| .output() |
| .expect("failed to SIGINT josh-proxy"); |
| // Sadly there is no "wait with timeout"... so we just give it some time to finish. |
| thread::sleep(Duration::from_millis(100)); |
| // Now hopefully it is gone. |
| if self |
| .0 |
| .try_wait() |
| .expect("failed to wait for josh-proxy") |
| .is_some() |
| { |
| return; |
| } |
| } |
| // If that didn't work (or we're not on Unix), kill it hard. |
| eprintln!( |
| "I have to kill josh-proxy the hard way, let's hope this does not \ |
| break anything." |
| ); |
| self.0.kill().expect("failed to SIGKILL josh-proxy"); |
| } |
| } |
