Source code for compiler_gym.envs.llvm.llvm_benchmark

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"""This module defines a utility function for constructing LLVM benchmarks."""
import logging
import os
import random
import subprocess
import sys
import tempfile
from concurrent.futures import as_completed
from datetime import datetime
from functools import lru_cache
from pathlib import Path
from typing import Iterable, List, Optional, Union

from compiler_gym.datasets import Benchmark
from compiler_gym.errors import BenchmarkInitError
from compiler_gym.third_party import llvm
from compiler_gym.util.commands import Popen, communicate, run_command
from compiler_gym.util.runfiles_path import transient_cache_path
from compiler_gym.util.shell_format import join_cmd
from compiler_gym.util.thread_pool import get_thread_pool_executor

logger = logging.getLogger(__name__)


class HostCompilerFailure(OSError):
    """Exception raised when the system compiler fails."""


class UnableToParseHostCompilerOutput(HostCompilerFailure):
    """Exception raised if unable to parse the verbose output of the host
    compiler."""


def _get_system_library_flags(compiler: str) -> Iterable[str]:
    """Private implementation function."""
    # Create a temporary file to write the compiled binary to, since GNU
    # assembler does not support piping to stdout.
    transient_cache = transient_cache_path(".")
    transient_cache.mkdir(parents=True, exist_ok=True)
    with tempfile.NamedTemporaryFile(dir=transient_cache) as f:
        cmd = [compiler, "-xc++", "-v", "-", "-o", f.name]
        # On macOS we need to compile a binary to invoke the linker.
        if sys.platform != "darwin":
            cmd.append("-c")

        # Retry loop to permit timeouts, though unlikely, in case of a
        # heavily overloaded system (I have observed CI failures because
        # of this).
        for _ in range(3):
            try:
                with Popen(
                    cmd,
                    stdout=subprocess.DEVNULL,
                    stderr=subprocess.PIPE,
                    stdin=subprocess.PIPE,
                    universal_newlines=True,
                ) as process:
                    _, stderr = communicate(
                        process=process, input="int main(){return 0;}", timeout=30
                    )
                    if process.returncode:
                        raise HostCompilerFailure(
                            f"Failed to invoke '{compiler}'. "
                            f"Is there a working system compiler?\n"
                            f"Error: {stderr.strip()}"
                        )
                    break
            except subprocess.TimeoutExpired:
                continue
            except FileNotFoundError as e:
                raise HostCompilerFailure(
                    f"Failed to invoke '{compiler}'. "
                    f"Is there a working system compiler?\n"
                    f"Error: {e}"
                ) from e
        else:
            raise HostCompilerFailure(
                f"Compiler invocation '{join_cmd(cmd)}' timed out after 3 attempts."
            )

    # Parse the compiler output that matches the conventional output format
    # used by clang and GCC:
    #
    #     #include <...> search starts here:
    #     /path/1
    #     /path/2
    #     End of search list
    in_search_list = False
    lines = stderr.split("\n")
    for line in lines:
        if in_search_list and line.startswith("End of search list"):
            break
        elif in_search_list:
            # We have an include path to return.
            path = Path(line.strip())
            yield "-isystem"
            yield str(path)
            # Compatibility fix for compiling benchmark sources which use the
            # '#include <endian.h>' header, which on macOS is located in a
            # 'machine/endian.h' directory.
            if (path / "machine").is_dir():
                yield "-isystem"
                yield str(path / "machine")
        elif line.startswith("#include <...> search starts here:"):
            in_search_list = True
    else:
        msg = f"Failed to parse '#include <...>' search paths from '{compiler}'"
        stderr = stderr.strip()
        if stderr:
            msg += f":\n{stderr}"
        raise UnableToParseHostCompilerOutput(msg)

    if sys.platform == "darwin":
        yield "-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"


@lru_cache(maxsize=32)
def _get_cached_system_library_flags(compiler: str) -> List[str]:
    """Private implementation detail."""
    return list(_get_system_library_flags(compiler))


[docs]def get_system_library_flags(compiler: Optional[str] = None) -> List[str]: """Determine the set of compilation flags needed to use the host system libraries. This uses the system compiler to determine the search paths for C/C++ system headers, and on macOS, the location of libclang_rt.osx.a. By default, :code:`c++` is invoked. This can be overridden by setting :code:`os.environ["CXX"]` prior to calling this function. :return: A list of command line flags for a compiler. :raises HostCompilerFailure: If the host compiler cannot be determined, or fails to compile a trivial piece of code. :raises UnableToParseHostCompilerOutput: If the output of the compiler cannot be understood. """ compiler = compiler or (os.environ.get("CXX") or "c++") # We want to cache the results of this expensive query after resolving the # default value for the compiler argument, as it can changed based on # environment variables. return _get_cached_system_library_flags(compiler)
[docs]class ClangInvocation: """Class to represent a single invocation of the clang compiler."""
[docs] def __init__( self, args: List[str], system_includes: bool = True, timeout: int = 600 ): """Create a clang invocation. :param args: The list of arguments to pass to clang. :param system_includes: Whether to include the system standard libraries during compilation jobs. This requires a system toolchain. See :func:`get_system_library_flags`. :param timeout: The maximum number of seconds to allow clang to run before terminating. """ self.args = args self.system_includes = system_includes self.timeout = timeout
def command(self, outpath: Path) -> List[str]: cmd = [str(llvm.clang_path()), "-c", "-emit-llvm", "-o", str(outpath)] if self.system_includes: cmd += get_system_library_flags() cmd += [str(s) for s in self.args] return cmd @classmethod def from_c_file( cls, path: Path, copt: Optional[List[str]] = None, system_includes: bool = True, timeout: int = 600, ) -> "ClangInvocation": copt = copt or [] # NOTE(cummins): There is some discussion about the best way to create a # bitcode that is unoptimized yet does not hinder downstream # optimization opportunities. Here we are using a configuration based on # -O1 in which we prevent the -O1 optimization passes from running. This # is because LLVM produces different function attributes dependening on # the optimization level. E.g. "-O0 -Xclang -disable-llvm-optzns -Xclang # -disable-O0-optnone" will generate code with "noinline" attributes set # on the functions, wheras "-Oz -Xclang -disable-llvm-optzns" will # generate functions with "minsize" and "optsize" attributes set. # # See also: # <https://lists.llvm.org/pipermail/llvm-dev/2018-August/thread.html#125365> # <https://github.com/facebookresearch/CompilerGym/issues/110> DEFAULT_COPT = [ "-O1", "-Xclang", "-disable-llvm-passes", "-Xclang", "-disable-llvm-optzns", ] return cls( DEFAULT_COPT + copt + [str(path)], system_includes=system_includes, timeout=timeout, )
[docs]def make_benchmark( inputs: Union[str, Path, ClangInvocation, List[Union[str, Path, ClangInvocation]]], copt: Optional[List[str]] = None, system_includes: bool = True, timeout: int = 600, ) -> Benchmark: """Create a benchmark for use by LLVM environments. This function takes one or more inputs and uses them to create an LLVM bitcode benchmark that can be passed to :meth:`compiler_gym.envs.LlvmEnv.reset`. The following input types are supported: +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | **File Suffix** | **Treated as** | **Converted using** | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | :code:`.bc` | LLVM IR bitcode | No conversion required. | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | :code:`.ll` | LLVM IR text format | Assembled to bitcode using llvm-as. | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ | :code:`.c`, :code:`.cc`, :code:`.cpp`, :code:`.cxx` | C / C++ source | Compiled to bitcode using clang and the given :code:`copt`. | +-----------------------------------------------------+---------------------+-------------------------------------------------------------+ .. note:: The LLVM IR format has no compatability guarantees between versions (see `LLVM docs <https://llvm.org/docs/DeveloperPolicy.html#ir-backwards-compatibility>`_). You must ensure that any :code:`.bc` and :code:`.ll` files are compatible with the LLVM version used by CompilerGym, which can be reported using :func:`env.compiler_version <compiler_gym.envs.CompilerEnv.compiler_version>`. E.g. for single-source C/C++ programs, you can pass the path of the source file: >>> benchmark = make_benchmark('my_app.c') >>> env = gym.make("llvm-v0") >>> env.reset(benchmark=benchmark) The clang invocation used is roughly equivalent to: .. code-block:: $ clang my_app.c -O0 -c -emit-llvm -o benchmark.bc Additional compile-time arguments to clang can be provided using the :code:`copt` argument: >>> benchmark = make_benchmark('/path/to/my_app.cpp', copt=['-O2']) If you need more fine-grained control over the options, you can directly construct a :class:`ClangInvocation <compiler_gym.envs.llvm.ClangInvocation>` to pass a list of arguments to clang: >>> benchmark = make_benchmark( ClangInvocation(['/path/to/my_app.c'], system_includes=False, timeout=10) ) For multi-file programs, pass a list of inputs that will be compiled separately and then linked to a single module: >>> benchmark = make_benchmark([ 'main.c', 'lib.cpp', 'lib2.bc', 'foo/input.bc' ]) :param inputs: An input, or list of inputs. :param copt: A list of command line options to pass to clang when compiling source files. :param system_includes: Whether to include the system standard libraries during compilation jobs. This requires a system toolchain. See :func:`get_system_library_flags`. :param timeout: The maximum number of seconds to allow clang to run before terminating. :return: A :code:`Benchmark` instance. :raises FileNotFoundError: If any input sources are not found. :raises TypeError: If the inputs are of unsupported types. :raises OSError: If a suitable compiler cannot be found. :raises BenchmarkInitError: If a compilation job fails. :raises TimeoutExpired: If a compilation job exceeds :code:`timeout` seconds. """ copt = copt or [] bitcodes: List[Path] = [] clang_jobs: List[ClangInvocation] = [] ll_paths: List[Path] = [] def _add_path(path: Path): if not path.is_file(): raise FileNotFoundError(path) if path.suffix == ".bc": bitcodes.append(path.absolute()) elif path.suffix in {".c", ".cc", ".cpp", ".cxx"}: clang_jobs.append( ClangInvocation.from_c_file( path, copt=copt, system_includes=system_includes, timeout=timeout ) ) elif path.suffix == ".ll": ll_paths.append(path) else: raise ValueError(f"Unrecognized file type: {path.name}") # Determine from inputs the list of pre-compiled bitcodes and the clang # invocations required to compile the bitcodes. if isinstance(inputs, str) or isinstance(inputs, Path): _add_path(Path(inputs)) elif isinstance(inputs, ClangInvocation): clang_jobs.append(inputs) else: for input in inputs: if isinstance(input, str) or isinstance(input, Path): _add_path(Path(input)) elif isinstance(input, ClangInvocation): clang_jobs.append(input) else: raise TypeError(f"Invalid input type: {type(input).__name__}") # Shortcut if we only have a single pre-compiled bitcode. if len(bitcodes) == 1 and not clang_jobs and not ll_paths: bitcode = bitcodes[0] return Benchmark.from_file(uri=f"benchmark://file-v0{bitcode}", path=bitcode) tmpdir_root = transient_cache_path(".") tmpdir_root.mkdir(exist_ok=True, parents=True) with tempfile.TemporaryDirectory( dir=tmpdir_root, prefix="llvm-make_benchmark-" ) as d: working_dir = Path(d) clang_outs = [ working_dir / f"clang-out-{i}.bc" for i in range(1, len(clang_jobs) + 1) ] llvm_as_outs = [ working_dir / f"llvm-as-out-{i}.bc" for i in range(1, len(ll_paths) + 1) ] # Run the clang and llvm-as invocations in parallel. Avoid running this # code path if possible as get_thread_pool_executor() requires locking. if clang_jobs or ll_paths: llvm_as_path = str(llvm.llvm_as_path()) executor = get_thread_pool_executor() llvm_as_commands = [ [llvm_as_path, str(ll_path), "-o", bc_path] for ll_path, bc_path in zip(ll_paths, llvm_as_outs) ] # Fire off the clang and llvm-as jobs. futures = [ executor.submit(run_command, job.command(out), job.timeout) for job, out in zip(clang_jobs, clang_outs) ] + [ executor.submit(run_command, command, timeout) for command in llvm_as_commands ] # Block until finished. list(future.result() for future in as_completed(futures)) # Check that the expected files were generated. for clang_job, bc_path in zip(clang_jobs, clang_outs): if not bc_path.is_file(): raise BenchmarkInitError( f"clang failed: {' '.join(clang_job.command(bc_path))}" ) for command, bc_path in zip(llvm_as_commands, llvm_as_outs): if not bc_path.is_file(): raise BenchmarkInitError(f"llvm-as failed: {command}") all_outs = bitcodes + clang_outs + llvm_as_outs if not all_outs: raise ValueError("No inputs") elif len(all_outs) == 1: # We only have a single bitcode so read it. with open(str(all_outs[0]), "rb") as f: bitcode = f.read() else: # Link all of the bitcodes into a single module. llvm_link_cmd = [str(llvm.llvm_link_path()), "-o", "-"] + [ str(path) for path in bitcodes + clang_outs ] with Popen( llvm_link_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) as llvm_link: bitcode, stderr = llvm_link.communicate(timeout=timeout) if llvm_link.returncode: raise BenchmarkInitError( f"Failed to link LLVM bitcodes with error: {stderr.decode('utf-8')}" ) timestamp = datetime.now().strftime("%Y%m%HT%H%M%S") uri = f"benchmark://user-v0/{timestamp}-{random.randrange(16**4):04x}" return Benchmark.from_file_contents(uri, bitcode)