312 lines
10 KiB
Python
312 lines
10 KiB
Python
|
"""Repository rule for Python autoconfiguration.
|
||
|
|
|
||
|
|
`python_configure` depends on the following environment variables:
|
||
|
|
|
||
|
|
* `PYTHON_BIN_PATH`: location of python binary.
|
||
|
|
* `PYTHON_LIB_PATH`: Location of python libraries.
|
||
|
|
"""
|
||
|
|
|
||
|
|
load(
|
||
|
|
"//third_party/remote_config:common.bzl",
|
||
|
|
"BAZEL_SH",
|
||
|
|
"PYTHON_BIN_PATH",
|
||
|
|
"PYTHON_LIB_PATH",
|
||
|
|
"TF_PYTHON_CONFIG_REPO",
|
||
|
|
"auto_config_fail",
|
||
|
|
"config_repo_label",
|
||
|
|
"execute",
|
||
|
|
"get_bash_bin",
|
||
|
|
"get_host_environ",
|
||
|
|
"get_python_bin",
|
||
|
|
"is_windows",
|
||
|
|
"raw_exec",
|
||
|
|
"read_dir",
|
||
|
|
)
|
||
|
|
|
||
|
|
def _genrule(src_dir, genrule_name, command, outs):
|
||
|
|
"""Returns a string with a genrule.
|
||
|
|
|
||
|
|
Genrule executes the given command and produces the given outputs.
|
||
|
|
"""
|
||
|
|
return (
|
||
|
|
"genrule(\n" +
|
||
|
|
' name = "' +
|
||
|
|
genrule_name + '",\n' +
|
||
|
|
" outs = [\n" +
|
||
|
|
outs +
|
||
|
|
"\n ],\n" +
|
||
|
|
' cmd = """\n' +
|
||
|
|
command +
|
||
|
|
'\n """,\n' +
|
||
|
|
")\n"
|
||
|
|
)
|
||
|
|
|
||
|
|
def _norm_path(path):
|
||
|
|
"""Returns a path with '/' and remove the trailing slash."""
|
||
|
|
path = path.replace("\\", "/")
|
||
|
|
if path[-1] == "/":
|
||
|
|
path = path[:-1]
|
||
|
|
return path
|
||
|
|
|
||
|
|
def _symlink_genrule_for_dir(
|
||
|
|
repository_ctx,
|
||
|
|
src_dir,
|
||
|
|
dest_dir,
|
||
|
|
genrule_name,
|
||
|
|
src_files = [],
|
||
|
|
dest_files = []):
|
||
|
|
"""Returns a genrule to symlink(or copy if on Windows) a set of files.
|
||
|
|
|
||
|
|
If src_dir is passed, files will be read from the given directory; otherwise
|
||
|
|
we assume files are in src_files and dest_files
|
||
|
|
"""
|
||
|
|
if src_dir != None:
|
||
|
|
src_dir = _norm_path(src_dir)
|
||
|
|
dest_dir = _norm_path(dest_dir)
|
||
|
|
files = "\n".join(read_dir(repository_ctx, src_dir))
|
||
|
|
|
||
|
|
# Create a list with the src_dir stripped to use for outputs.
|
||
|
|
dest_files = files.replace(src_dir, "").splitlines()
|
||
|
|
src_files = files.splitlines()
|
||
|
|
command = []
|
||
|
|
outs = []
|
||
|
|
for i in range(len(dest_files)):
|
||
|
|
if dest_files[i] != "":
|
||
|
|
# If we have only one file to link we do not want to use the dest_dir, as
|
||
|
|
# $(@D) will include the full path to the file.
|
||
|
|
dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i]
|
||
|
|
|
||
|
|
# Copy the headers to create a sandboxable setup.
|
||
|
|
cmd = "cp -f"
|
||
|
|
command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
|
||
|
|
outs.append(' "' + dest_dir + dest_files[i] + '",')
|
||
|
|
genrule = _genrule(
|
||
|
|
src_dir,
|
||
|
|
genrule_name,
|
||
|
|
" && ".join(command),
|
||
|
|
"\n".join(outs),
|
||
|
|
)
|
||
|
|
return genrule
|
||
|
|
|
||
|
|
def _get_python_lib(repository_ctx, python_bin):
|
||
|
|
"""Gets the python lib path."""
|
||
|
|
python_lib = get_host_environ(repository_ctx, PYTHON_LIB_PATH)
|
||
|
|
if python_lib != None:
|
||
|
|
return python_lib
|
||
|
|
|
||
|
|
# The interesting program to execute.
|
||
|
|
print_lib = [
|
||
|
|
"from __future__ import print_function",
|
||
|
|
"import site",
|
||
|
|
"import os",
|
||
|
|
"python_paths = []",
|
||
|
|
"if os.getenv('PYTHONPATH') is not None:",
|
||
|
|
" python_paths = os.getenv('PYTHONPATH').split(':')",
|
||
|
|
"try:",
|
||
|
|
" library_paths = site.getsitepackages()",
|
||
|
|
"except AttributeError:",
|
||
|
|
" from distutils.sysconfig import get_python_lib",
|
||
|
|
" library_paths = [get_python_lib()]",
|
||
|
|
"all_paths = set(python_paths + library_paths)",
|
||
|
|
"paths = []",
|
||
|
|
"for path in all_paths:",
|
||
|
|
" if os.path.isdir(path):",
|
||
|
|
" paths.append(path)",
|
||
|
|
"if len(paths) >=1:",
|
||
|
|
" print(paths[0])",
|
||
|
|
]
|
||
|
|
|
||
|
|
# The below script writes the above program to a file
|
||
|
|
# and executes it. This is to work around the limitation
|
||
|
|
# of not being able to upload files as part of execute.
|
||
|
|
cmd = "from os import linesep;"
|
||
|
|
cmd += "f = open('script.py', 'w');"
|
||
|
|
for line in print_lib:
|
||
|
|
cmd += "f.write(\"%s\" + linesep);" % line
|
||
|
|
cmd += "f.close();"
|
||
|
|
cmd += "from os import system;"
|
||
|
|
cmd += "system(\"%s script.py\");" % python_bin
|
||
|
|
|
||
|
|
result = execute(repository_ctx, [python_bin, "-c", cmd])
|
||
|
|
return result.stdout.strip()
|
||
|
|
|
||
|
|
def _check_python_lib(repository_ctx, python_lib):
|
||
|
|
"""Checks the python lib path."""
|
||
|
|
cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
|
||
|
|
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
|
||
|
|
if result.return_code == 1:
|
||
|
|
auto_config_fail("Invalid python library path: %s" % python_lib)
|
||
|
|
|
||
|
|
def _check_python_bin(repository_ctx, python_bin):
|
||
|
|
"""Checks the python bin path."""
|
||
|
|
cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
|
||
|
|
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd])
|
||
|
|
if result.return_code == 1:
|
||
|
|
auto_config_fail("--define %s='%s' is not executable. Is it the python binary?" % (
|
||
|
|
PYTHON_BIN_PATH,
|
||
|
|
python_bin,
|
||
|
|
))
|
||
|
|
|
||
|
|
def _get_python_include(repository_ctx, python_bin):
|
||
|
|
"""Gets the python include path."""
|
||
|
|
result = execute(
|
||
|
|
repository_ctx,
|
||
|
|
[
|
||
|
|
python_bin,
|
||
|
|
"-c",
|
||
|
|
"from __future__ import print_function;" +
|
||
|
|
"from distutils import sysconfig;" +
|
||
|
|
"print(sysconfig.get_python_inc())",
|
||
|
|
],
|
||
|
|
error_msg = "Problem getting python include path.",
|
||
|
|
error_details = ("Is the Python binary path set up right? " +
|
||
|
|
"(See ./configure or " + PYTHON_BIN_PATH + ".) " +
|
||
|
|
"Is distutils installed?"),
|
||
|
|
)
|
||
|
|
return result.stdout.splitlines()[0]
|
||
|
|
|
||
|
|
def _get_python_import_lib_name(repository_ctx, python_bin):
|
||
|
|
"""Get Python import library name (pythonXY.lib) on Windows."""
|
||
|
|
result = execute(
|
||
|
|
repository_ctx,
|
||
|
|
[
|
||
|
|
python_bin,
|
||
|
|
"-c",
|
||
|
|
"import sys;" +
|
||
|
|
'print("python" + str(sys.version_info[0]) + ' +
|
||
|
|
' str(sys.version_info[1]) + ".lib")',
|
||
|
|
],
|
||
|
|
error_msg = "Problem getting python import library.",
|
||
|
|
error_details = ("Is the Python binary path set up right? " +
|
||
|
|
"(See ./configure or " + PYTHON_BIN_PATH + ".) "),
|
||
|
|
)
|
||
|
|
return result.stdout.splitlines()[0]
|
||
|
|
|
||
|
|
def _get_numpy_include(repository_ctx, python_bin):
|
||
|
|
"""Gets the numpy include path."""
|
||
|
|
return execute(
|
||
|
|
repository_ctx,
|
||
|
|
[
|
||
|
|
python_bin,
|
||
|
|
"-c",
|
||
|
|
"from __future__ import print_function;" +
|
||
|
|
"import numpy;" +
|
||
|
|
" print(numpy.get_include());",
|
||
|
|
],
|
||
|
|
error_msg = "Problem getting numpy include path.",
|
||
|
|
error_details = "Is numpy installed?",
|
||
|
|
).stdout.splitlines()[0]
|
||
|
|
|
||
|
|
def _create_local_python_repository(repository_ctx):
|
||
|
|
"""Creates the repository containing files set up to build with Python."""
|
||
|
|
|
||
|
|
# Resolve all labels before doing any real work. Resolving causes the
|
||
|
|
# function to be restarted with all previous state being lost. This
|
||
|
|
# can easily lead to a O(n^2) runtime in the number of labels.
|
||
|
|
build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl"))
|
||
|
|
|
||
|
|
python_bin = get_python_bin(repository_ctx)
|
||
|
|
_check_python_bin(repository_ctx, python_bin)
|
||
|
|
python_lib = _get_python_lib(repository_ctx, python_bin)
|
||
|
|
_check_python_lib(repository_ctx, python_lib)
|
||
|
|
python_include = _get_python_include(repository_ctx, python_bin)
|
||
|
|
numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy"
|
||
|
|
python_include_rule = _symlink_genrule_for_dir(
|
||
|
|
repository_ctx,
|
||
|
|
python_include,
|
||
|
|
"python_include",
|
||
|
|
"python_include",
|
||
|
|
)
|
||
|
|
python_import_lib_genrule = ""
|
||
|
|
|
||
|
|
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
|
||
|
|
# See https://docs.python.org/3/extending/windows.html
|
||
|
|
if is_windows(repository_ctx):
|
||
|
|
python_include = _norm_path(python_include)
|
||
|
|
python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin)
|
||
|
|
python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name
|
||
|
|
python_import_lib_genrule = _symlink_genrule_for_dir(
|
||
|
|
repository_ctx,
|
||
|
|
None,
|
||
|
|
"",
|
||
|
|
"python_import_lib",
|
||
|
|
[python_import_lib_src],
|
||
|
|
[python_import_lib_name],
|
||
|
|
)
|
||
|
|
numpy_include_rule = _symlink_genrule_for_dir(
|
||
|
|
repository_ctx,
|
||
|
|
numpy_include,
|
||
|
|
"numpy_include/numpy",
|
||
|
|
"numpy_include",
|
||
|
|
)
|
||
|
|
|
||
|
|
platform_constraint = ""
|
||
|
|
if repository_ctx.attr.platform_constraint:
|
||
|
|
platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint
|
||
|
|
repository_ctx.template("BUILD", build_tpl, {
|
||
|
|
"%{PYTHON_BIN_PATH}": python_bin,
|
||
|
|
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
|
||
|
|
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
|
||
|
|
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
|
||
|
|
"%{PLATFORM_CONSTRAINT}": platform_constraint,
|
||
|
|
})
|
||
|
|
|
||
|
|
def _create_remote_python_repository(repository_ctx, remote_config_repo):
|
||
|
|
"""Creates pointers to a remotely configured repo set up to build with Python.
|
||
|
|
"""
|
||
|
|
repository_ctx.template("BUILD", config_repo_label(remote_config_repo, ":BUILD"), {})
|
||
|
|
|
||
|
|
def _python_autoconf_impl(repository_ctx):
|
||
|
|
"""Implementation of the python_autoconf repository rule."""
|
||
|
|
if get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO) != None:
|
||
|
|
_create_remote_python_repository(
|
||
|
|
repository_ctx,
|
||
|
|
get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO),
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
_create_local_python_repository(repository_ctx)
|
||
|
|
|
||
|
|
_ENVIRONS = [
|
||
|
|
BAZEL_SH,
|
||
|
|
PYTHON_BIN_PATH,
|
||
|
|
PYTHON_LIB_PATH,
|
||
|
|
]
|
||
|
|
|
||
|
|
local_python_configure = repository_rule(
|
||
|
|
implementation = _create_local_python_repository,
|
||
|
|
environ = _ENVIRONS,
|
||
|
|
attrs = {
|
||
|
|
"environ": attr.string_dict(),
|
||
|
|
"platform_constraint": attr.string(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
|
||
|
|
remote_python_configure = repository_rule(
|
||
|
|
implementation = _create_local_python_repository,
|
||
|
|
environ = _ENVIRONS,
|
||
|
|
remotable = True,
|
||
|
|
attrs = {
|
||
|
|
"environ": attr.string_dict(),
|
||
|
|
"platform_constraint": attr.string(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
|
||
|
|
python_configure = repository_rule(
|
||
|
|
implementation = _python_autoconf_impl,
|
||
|
|
environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO],
|
||
|
|
attrs = {
|
||
|
|
"platform_constraint": attr.string(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
"""Detects and configures the local Python.
|
||
|
|
|
||
|
|
Add the following to your WORKSPACE FILE:
|
||
|
|
|
||
|
|
```python
|
||
|
|
python_configure(name = "local_config_python")
|
||
|
|
```
|
||
|
|
|
||
|
|
Args:
|
||
|
|
name: A unique name for this workspace rule.
|
||
|
|
"""
|