src/ci/docker/scripts/android-sdk-manager.py

#!/usr/bin/env python3
# Simpler reimplementation of Android's sdkmanager
# Extra features of this implementation are pinning and mirroring

import argparse
import hashlib
import os
import subprocess
import tempfile
import urllib.request
import xml.etree.ElementTree as ET

# These URLs are the Google repositories containing the list of available
# packages and their versions. The list has been generated by listing the URLs
# fetched while executing `tools/bin/sdkmanager --list`
BASE_REPOSITORY = "https://dl.google.com/android/repository/"
REPOSITORIES = [
    "sys-img/android/sys-img2-1.xml",
    "sys-img/android-wear/sys-img2-1.xml",
    "sys-img/android-wear-cn/sys-img2-1.xml",
    "sys-img/android-tv/sys-img2-1.xml",
    "sys-img/google_apis/sys-img2-1.xml",
    "sys-img/google_apis_playstore/sys-img2-1.xml",
    "addon2-1.xml",
    "glass/addon2-1.xml",
    "extras/intel/addon2-1.xml",
    "repository2-1.xml",
]

# Available hosts: linux, macosx and windows
HOST_OS = "linux"

# Mirroring options
MIRROR_BUCKET = "rust-lang-ci-mirrors"
MIRROR_BUCKET_REGION = "us-west-1"
MIRROR_BASE_DIR = "rustc/android/"


class Package:
    def __init__(self, path, url, sha1, deps=None):
        if deps is None:
            deps = []
        self.path = path.strip()
        self.url = url.strip()
        self.sha1 = sha1.strip()
        self.deps = deps

    def download(self, base_url):
        _, file = tempfile.mkstemp()
        url = base_url + self.url
        subprocess.run(["curl", "-o", file, url], check=True)
        # Ensure there are no hash mismatches
        with open(file, "rb") as f:
            sha1 = hashlib.sha1(f.read()).hexdigest()
            if sha1 != self.sha1:
                raise RuntimeError(
                    "hash mismatch for package "
                    + self.path
                    + ": "
                    + sha1
                    + " vs "
                    + self.sha1
                    + " (known good)"
                )
        return file

    def __repr__(self):
        return "<Package " + self.path + " at " + self.url + " (sha1=" + self.sha1 + ")"


def fetch_url(url):
    page = urllib.request.urlopen(url)
    return page.read()


def fetch_repository(base, repo_url):
    packages = {}
    root = ET.fromstring(fetch_url(base + repo_url))
    for package in root:
        if package.tag != "remotePackage":
            continue
        path = package.attrib["path"]

        for archive in package.find("archives"):
            host_os = archive.find("host-os")
            if host_os is not None and host_os.text != HOST_OS:
                continue
            complete = archive.find("complete")
            url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
            sha1 = complete.find("checksum").text

            deps = []
            dependencies = package.find("dependencies")
            if dependencies is not None:
                for dep in dependencies:
                    deps.append(dep.attrib["path"])

            packages[path] = Package(path, url, sha1, deps)
            break

    return packages


def fetch_repositories():
    packages = {}
    for repo in REPOSITORIES:
        packages.update(fetch_repository(BASE_REPOSITORY, repo))
    return packages


class Lockfile:
    def __init__(self, path):
        self.path = path
        self.packages = {}
        if os.path.exists(path):
            with open(path) as f:
                for line in f:
                    path, url, sha1 = line.split(" ")
                    self.packages[path] = Package(path, url, sha1)

    def add(self, packages, name, *, update=True):
        if name not in packages:
            raise NameError("package not found: " + name)
        if not update and name in self.packages:
            return
        self.packages[name] = packages[name]
        for dep in packages[name].deps:
            self.add(packages, dep, update=False)

    def save(self):
        packages = list(sorted(self.packages.values(), key=lambda p: p.path))
        with open(self.path, "w") as f:
            for package in packages:
                f.write(package.path + " " + package.url + " " + package.sha1 + "\n")


def cli_add_to_lockfile(args):
    lockfile = Lockfile(args.lockfile)
    packages = fetch_repositories()
    for package in args.packages:
        lockfile.add(packages, package)
    lockfile.save()


def cli_update_mirror(args):
    lockfile = Lockfile(args.lockfile)
    for package in lockfile.packages.values():
        path = package.download(BASE_REPOSITORY)
        subprocess.run(
            [
                "aws",
                "s3",
                "mv",
                path,
                "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
                "--profile=" + args.awscli_profile,
            ],
            check=True,
        )


def cli_install(args):
    lockfile = Lockfile(args.lockfile)
    for package in lockfile.packages.values():
        # Download the file from the mirror into a temp file
        url = (
            "https://"
            + MIRROR_BUCKET
            + ".s3-"
            + MIRROR_BUCKET_REGION
            + ".amazonaws.com/"
            + MIRROR_BASE_DIR
        )
        downloaded = package.download(url)
        # Extract the file in a temporary directory
        extract_dir = tempfile.mkdtemp()
        subprocess.run(
            [
                "unzip",
                "-q",
                downloaded,
                "-d",
                extract_dir,
            ],
            check=True,
        )
        # Figure out the prefix used in the zip
        subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
        if len(subdirs) != 1:
            raise RuntimeError("extracted directory contains more than one dir")
        # Move the extracted files in the proper directory
        dest = os.path.join(args.dest, package.path.replace(";", "/"))
        os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
        os.rename(os.path.join(extract_dir, subdirs[0]), dest)
        os.unlink(downloaded)


def cli():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    add_to_lockfile = subparsers.add_parser("add-to-lockfile")
    add_to_lockfile.add_argument("lockfile")
    add_to_lockfile.add_argument("packages", nargs="+")
    add_to_lockfile.set_defaults(func=cli_add_to_lockfile)

    update_mirror = subparsers.add_parser("update-mirror")
    update_mirror.add_argument("lockfile")
    update_mirror.add_argument("--awscli-profile", default="default")
    update_mirror.set_defaults(func=cli_update_mirror)

    install = subparsers.add_parser("install")
    install.add_argument("lockfile")
    install.add_argument("dest")
    install.set_defaults(func=cli_install)

    args = parser.parse_args()
    if not hasattr(args, "func"):
        print("error: a subcommand is required (see --help)")
        exit(1)
    args.func(args)


if __name__ == "__main__":
    cli()