| #!/usr/bin/env python3 |
| """Create a text file listing all public API. This can be used to ensure that all |
| functions are covered by our macros. |
| |
| This file additionally does tidy-esque checks that all functions are listed where |
| needed, or that lists are sorted. |
| """ |
| |
| import difflib |
| import json |
| import re |
| import subprocess as sp |
| import sys |
| from dataclasses import dataclass |
| from glob import glob |
| from pathlib import Path |
| from typing import Any, Callable, TypeAlias |
| |
| SELF_PATH = Path(__file__) |
| ETC_DIR = SELF_PATH.parent |
| ROOT_DIR = ETC_DIR.parent |
| |
| # These files do not trigger a retest. |
| IGNORED_SOURCES = ["libm/src/libm_helper.rs", "libm/src/math/support/float_traits.rs"] |
| |
| IndexTy: TypeAlias = dict[str, dict[str, Any]] |
| """Type of the `index` item in rustdoc's JSON output""" |
| |
| |
| def eprint(*args, **kwargs): |
| """Print to stderr.""" |
| print(*args, file=sys.stderr, **kwargs) |
| |
| |
| @dataclass |
| class Crate: |
| """Representation of public interfaces and function definition locations in |
| `libm`. |
| """ |
| |
| public_functions: list[str] |
| """List of all public functions.""" |
| defs: dict[str, list[str]] |
| """Map from `name->[source files]` to find all places that define a public |
| function. We track this to know which tests need to be rerun when specific files |
| get updated. |
| """ |
| types: dict[str, str] |
| """Map from `name->type`.""" |
| |
| def __init__(self) -> None: |
| self.public_functions = [] |
| self.defs = {} |
| self.types = {} |
| |
| j = self.get_rustdoc_json() |
| index: IndexTy = j["index"] |
| self._init_function_list(index) |
| self._init_defs(index) |
| self._init_types() |
| |
| @staticmethod |
| def get_rustdoc_json() -> dict[Any, Any]: |
| """Get rustdoc's JSON output for the `libm` crate.""" |
| |
| j = sp.check_output( |
| [ |
| "rustdoc", |
| "libm/src/lib.rs", |
| "--edition=2021", |
| "--document-private-items", |
| "--output-format=json", |
| "--cfg=f16_enabled", |
| "--cfg=f128_enabled", |
| "-Zunstable-options", |
| "-o-", |
| ], |
| cwd=ROOT_DIR, |
| text=True, |
| ) |
| j = json.loads(j) |
| return j |
| |
| def _init_function_list(self, index: IndexTy) -> None: |
| """Get a list of public functions from rustdoc JSON output. |
| |
| Note that this only finds functions that are reexported in `lib.rs`, this will |
| need to be adjusted if we need to account for functions that are defined there, or |
| glob reexports in other locations. |
| """ |
| # Filter out items that are not public |
| public = [i for i in index.values() if i["visibility"] == "public"] |
| |
| # Collect a list of source IDs for reexported items in `lib.rs` or `mod math`. |
| use = (i for i in public if "use" in i["inner"]) |
| use = ( |
| i |
| for i in use |
| if i["span"]["filename"] in ["libm/src/math/mod.rs", "libm/src/lib.rs"] |
| ) |
| reexported_ids = [item["inner"]["use"]["id"] for item in use] |
| |
| # Collect a list of reexported items that are functions |
| for id in reexported_ids: |
| srcitem = index.get(str(id)) |
| # External crate |
| if srcitem is None: |
| continue |
| |
| # Skip if not a function |
| if "function" not in srcitem["inner"]: |
| continue |
| |
| self.public_functions.append(srcitem["name"]) |
| self.public_functions.sort() |
| |
| def _init_defs(self, index: IndexTy) -> None: |
| defs = {name: set() for name in self.public_functions} |
| funcs = (i for i in index.values() if "function" in i["inner"]) |
| funcs = (f for f in funcs if f["name"] in self.public_functions) |
| for func in funcs: |
| defs[func["name"]].add(func["span"]["filename"]) |
| |
| # A lot of the `arch` module is often configured out so doesn't show up in docs. Use |
| # string matching as a fallback. |
| for fname in glob( |
| "libm/src/math/arch/**/*.rs", root_dir=ROOT_DIR, recursive=True |
| ): |
| contents = (ROOT_DIR.joinpath(fname)).read_text() |
| |
| for name in self.public_functions: |
| if f"fn {name}" in contents: |
| defs[name].add(fname) |
| |
| for name, sources in defs.items(): |
| base_sources = defs[base_name(name)[0]] |
| for src in (s for s in base_sources if "generic" in s): |
| sources.add(src) |
| |
| for src in IGNORED_SOURCES: |
| sources.discard(src) |
| |
| # Sort the set |
| self.defs = {k: sorted(v) for (k, v) in defs.items()} |
| |
| def _init_types(self) -> None: |
| self.types = {name: base_name(name)[1] for name in self.public_functions} |
| |
| def write_function_list(self, check: bool) -> None: |
| """Collect the list of public functions to a simple text file.""" |
| output = "# autogenerated by update-api-list.py\n" |
| for name in self.public_functions: |
| output += f"{name}\n" |
| |
| out_file = ETC_DIR.joinpath("function-list.txt") |
| |
| if check: |
| with open(out_file, "r") as f: |
| current = f.read() |
| diff_and_exit(current, output, "function list") |
| else: |
| with open(out_file, "w") as f: |
| f.write(output) |
| |
| def write_function_defs(self, check: bool) -> None: |
| """Collect the list of information about public functions to a JSON file .""" |
| comment = ( |
| "Autogenerated by update-api-list.py. " |
| "List of files that define a function with a given name. " |
| "This file is checked in to make it obvious if refactoring breaks things" |
| ) |
| |
| d = {"__comment": comment} |
| d |= { |
| name: {"sources": self.defs[name], "type": self.types[name]} |
| for name in self.public_functions |
| } |
| |
| out_file = ETC_DIR.joinpath("function-definitions.json") |
| output = json.dumps(d, indent=4) + "\n" |
| |
| if check: |
| with open(out_file, "r") as f: |
| current = f.read() |
| diff_and_exit(current, output, "source list") |
| else: |
| with open(out_file, "w") as f: |
| f.write(output) |
| |
| def tidy_lists(self) -> None: |
| """In each file, check annotations indicating blocks of code should be sorted or should |
| include all public API. |
| """ |
| |
| flist = sp.check_output(["git", "ls-files"], cwd=ROOT_DIR, text=True) |
| |
| for path in flist.splitlines(): |
| fpath = ROOT_DIR.joinpath(path) |
| if fpath.is_dir() or fpath == SELF_PATH: |
| continue |
| |
| lines = fpath.read_text().splitlines() |
| |
| validate_delimited_block( |
| fpath, |
| lines, |
| "verify-sorted-start", |
| "verify-sorted-end", |
| ensure_sorted, |
| ) |
| |
| validate_delimited_block( |
| fpath, |
| lines, |
| "verify-apilist-start", |
| "verify-apilist-end", |
| lambda p, n, lines: self.ensure_contains_api(p, n, lines), |
| ) |
| |
| def ensure_contains_api(self, fpath: Path, line_num: int, lines: list[str]): |
| """Given a list of strings, ensure that each public function we have is named |
| somewhere. |
| """ |
| not_found = [] |
| for func in self.public_functions: |
| # The function name may be on its own or somewhere in a snake case string. |
| pat = re.compile(rf"(\b|_){func}(\b|_)") |
| found = next((line for line in lines if pat.search(line)), None) |
| |
| if found is None: |
| not_found.append(func) |
| |
| if len(not_found) == 0: |
| return |
| |
| relpath = fpath.relative_to(ROOT_DIR) |
| eprint(f"functions not found at {relpath}:{line_num}: {not_found}") |
| exit(1) |
| |
| |
| def validate_delimited_block( |
| fpath: Path, |
| lines: list[str], |
| start: str, |
| end: str, |
| validate: Callable[[Path, int, list[str]], None], |
| ) -> None: |
| """Identify blocks of code wrapped within `start` and `end`, collect their contents |
| to a list of strings, and call `validate` for each of those lists. |
| """ |
| relpath = fpath.relative_to(ROOT_DIR) |
| block_lines = [] |
| block_start_line: None | int = None |
| for line_num, line in enumerate(lines): |
| line_num += 1 |
| |
| if start in line: |
| block_start_line = line_num |
| continue |
| |
| if end in line: |
| if block_start_line is None: |
| eprint(f"`{end}` without `{start}` at {relpath}:{line_num}") |
| exit(1) |
| |
| validate(fpath, block_start_line, block_lines) |
| block_lines = [] |
| block_start_line = None |
| continue |
| |
| if block_start_line is not None: |
| block_lines.append(line) |
| |
| if block_start_line is not None: |
| eprint(f"`{start}` without `{end}` at {relpath}:{block_start_line}") |
| exit(1) |
| |
| |
| def ensure_sorted(fpath: Path, block_start_line: int, lines: list[str]) -> None: |
| """Ensure that a list of lines is sorted, otherwise print a diff and exit.""" |
| relpath = fpath.relative_to(ROOT_DIR) |
| diff_and_exit( |
| "\n".join(lines), |
| "\n".join(sorted(lines)), |
| f"sorted block at {relpath}:{block_start_line}", |
| ) |
| |
| |
| def diff_and_exit(actual: str, expected: str, name: str): |
| """If the two strings are different, print a diff between them and then exit |
| with an error. |
| """ |
| if actual == expected: |
| print(f"{name} output matches expected; success") |
| return |
| |
| a = [f"{line}\n" for line in actual.splitlines()] |
| b = [f"{line}\n" for line in expected.splitlines()] |
| |
| diff = difflib.unified_diff(a, b, "actual", "expected") |
| sys.stdout.writelines(diff) |
| print(f"mismatched {name}") |
| exit(1) |
| |
| |
| def base_name(name: str) -> tuple[str, str]: |
| """Return the basename and type from a full function name. Keep in sync with Rust's |
| `fn base_name`. |
| """ |
| known_mappings = [ |
| ("erff", ("erf", "f32")), |
| ("erf", ("erf", "f64")), |
| ("modff", ("modf", "f32")), |
| ("modf", ("modf", "f64")), |
| ("lgammaf_r", ("lgamma_r", "f32")), |
| ("lgamma_r", ("lgamma_r", "f64")), |
| ] |
| |
| found = next((base for (full, base) in known_mappings if full == name), None) |
| if found is not None: |
| return found |
| |
| if name.endswith("f"): |
| return (name.rstrip("f"), "f32") |
| |
| if name.endswith("f16"): |
| return (name.rstrip("f16"), "f16") |
| |
| if name.endswith("f128"): |
| return (name.rstrip("f128"), "f128") |
| |
| return (name, "f64") |
| |
| |
| def ensure_updated_list(check: bool) -> None: |
| """Runner to update the function list and JSON, or check that it is already up |
| to date. |
| """ |
| crate = Crate() |
| crate.write_function_list(check) |
| crate.write_function_defs(check) |
| |
| crate.tidy_lists() |
| |
| |
| def main(): |
| """By default overwrite the file. If `--check` is passed, print a diff instead and |
| error if the files are different. |
| """ |
| match sys.argv: |
| case [_]: |
| ensure_updated_list(False) |
| case [_, "--check"]: |
| ensure_updated_list(True) |
| case _: |
| print("unrecognized arguments") |
| exit(1) |
| |
| |
| if __name__ == "__main__": |
| main() |
