"""
Common subroutines for consistency between gitlab-ci / github actions / etc...
"""
import ubelt as ub
[docs]
def make_typecheck_parts(self):
"""
Return a list of shell commands to run type checkers.
By default this will run both `mypy` and `ty` (in that order). The
returned value is a list of command strings so callers can adapt it to
either GitHub Actions (`run` string) or GitLab CI (`script` list).
"""
# TODO: more control over which type checkers to use.
# right now we always enable ty unless notypes is on.
# but we should have more sane defaults.
checkers = None
if checkers is None:
checkers = ['ty']
if 'mypy' in self.tags:
checkers += ['mypy']
# Where to install runtime/type requirements from
type_requirement_files = [
# TODO: get this location from the config
'requirements/runtime.txt'
]
req_files_text = ' '.join(type_requirement_files)
if self.config['use_pyproject_requirements']:
pip_install_reqs = 'pip install -r pyproject.toml'
else:
pip_install_reqs = f'pip install -r {req_files_text}'
commands = []
if 'mypy' in checkers:
commands += [
'python -m pip install mypy',
pip_install_reqs,
# TODO; this likely needs to be replaced with some explicit
# registration of what typing requirements are for the library
# f'mypy --install-types --non-interactive ./{self.rel_mod_dpath}',
f'mypy ./{self.rel_mod_dpath}',
]
if 'ty' in checkers:
# Generic support for "ty". Install and run; users can customize
# behavior by changing `checkers` or adding config-specific steps.
commands += [
'python -m pip install ty',
pip_install_reqs,
f'ty check ./{self.rel_mod_dpath}',
]
return commands
[docs]
def make_build_sdist_parts(self, wheelhouse_dpath='wheelhouse'):
commands = [
# 'python -m pip install pip -U',
f'{self.UPDATE_PIP}',
f'{self.PIP_INSTALL} setuptools>=0.8 wheel build twine',
f'python -m build --sdist --outdir {wheelhouse_dpath}',
f'python -m twine check ./{wheelhouse_dpath}/{self.pkg_fname_prefix}*.tar.gz',
]
build_parts = {
'commands': commands,
'artifact': f'./{wheelhouse_dpath}/{self.pkg_fname_prefix}*.tar.gz',
}
return build_parts
[docs]
def make_build_wheel_parts(self, wheelhouse_dpath='wheelhouse'):
commands = [
# 'python -m pip install pip -U',
f'{self.UPDATE_PIP}',
f'{self.PIP_INSTALL} setuptools>=0.8 wheel build twine',
f'python -m build --wheel --outdir {wheelhouse_dpath}',
f'python -m twine check ./{wheelhouse_dpath}/{self.pkg_fname_prefix}*.whl',
]
build_wheel_parts = {
'commands': commands,
'artifact': f'./{wheelhouse_dpath}/{self.pkg_fname_prefix}*.whl',
}
return build_wheel_parts
[docs]
def make_install_and_test_wheel_parts(
self,
wheelhouse_dpath,
special_install_lines,
workspace_dname,
custom_before_test_lines=[],
custom_after_test_commands=[],
):
"""
Builds the YAML common between github actions and gitlab CI to install and
tests python packages.
References:
https://stackoverflow.com/questions/42019184/python-how-can-i-get-the-version-number-from-a-whl-file
"""
from xcookie.util_yaml import Yaml
# get_modname_python = "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['tool']['xcookie']['mod_name'])"
# get_modname_bash = f'python -c "{get_modname_python}"'
# get_wheel_fpath_python = f"import pathlib; print(str(sorted(pathlib.Path('{wheelhouse_dpath}').glob('{self.mod_name}*.whl'))[-1]).replace(r'\\', '/'))"
# get_wheel_fpath_python = Yaml.CodeBlock(f"import pathlib; print(str(sorted(pathlib.Path('{wheelhouse_dpath}').glob('{self.mod_name}*.whl'))[-1]).replace(chr(92), chr(47)))")
# get_wheel_fpath_bash = f'python -c "{get_wheel_fpath_python}"'
# get_mod_version_python = "from pkginfo import Wheel; print(Wheel('$WHEEL_FPATH').version)"
# get_mod_version_bash = f'python -c "{get_mod_version_python}"'
get_wheel_fpath_bash = ub.codeblock(
f"""
python -c "if 1:
import pathlib
from packaging import tags
from packaging.utils import parse_wheel_filename
dist_dpath = pathlib.Path('{wheelhouse_dpath}')
wheels = sorted(dist_dpath.glob('{self.pkg_fname_prefix}*.whl'))
if wheels:
sys_tags = set(tags.sys_tags())
matching = []
for w in wheels:
try:
_, _, _, wheel_tags = parse_wheel_filename(w.name)
except Exception:
continue
if any(t in sys_tags for t in wheel_tags):
matching.append(w)
fpath = sorted(matching or wheels)[-1]
else:
sdists = sorted(dist_dpath.glob('{self.pkg_fname_prefix}*.tar.gz'))
if not sdists:
raise SystemExit('No wheel artifacts found in wheelhouse')
fpath = sdists[-1]
print(str(fpath).replace(chr(92), chr(47)))
"
"""
)
# if tuple(map(int, self.config.min_python.split('.'))) >= (3, 8):
# # Not sure why this fails on 3.6 / 3.7?
# # Use less ugly version when we can
# get_mod_version_bash = ub.codeblock(
# """
# python -c "if 1:
# from pkginfo import Wheel, SDist
# import pathlib
# fpath = '$WHEEL_FPATH'
# cls = Wheel if fpath.endswith('.whl') else SDist
# item = cls(fpath)
# print(item.version)
# "
# """
# )
# else:
# get_mod_version_bash = ub.codeblock(
# """
# python -c "if 1:
# from pkginfo import Wheel, SDist
# import pathlib
# fpath = '$WHEEL_FPATH'
# cls = Wheel if fpath.endswith('.whl') else SDist
# item = cls(fpath)
# if item.version is None:
# import re
# # This is very fragile
# fname = pathlib.Path(fpath).name
# match = re.match(r'^([^-]+)-([^-]+)(.whl|.tar.gz)$', fname)
# bs = chr(92)
# pat = '([0-9]+' + bs + '.[0-9]+' + bs + '.[0-9]+)'
# import re
# # Not sure why version is None in 3.6 and 3.7
# match = re.search(pat, fname)
# assert match is not None
# version = match.groups()[0]
# print(version)
# else:
# print(item.version)
# "
# """
# )
# get_modpath_python = "import ubelt; print(ubelt.modname_to_modpath(f'{self.mod_name}'))"
get_modpath_python = f'import {self.mod_name}, os; print(os.path.dirname({self.mod_name}.__file__))'
get_modpath_bash = f'python -c "{get_modpath_python}"'
test_command = self.config['test_command']
if test_command == 'auto':
if 'ibeis' == self.mod_name:
test_command = [
'python -m xdoctest $MOD_DPATH --style=google all',
'echo "xdoctest command finished"',
]
else:
test_command = [
Yaml.CodeBlock(
'python -m pytest --verbose -p pytester -p no:doctest --xdoctest --cov-config ../pyproject.toml --cov-report term --durations=100 --cov="$MOD_NAME" "$MOD_DPATH" ../tests'
),
'echo "pytest command finished, moving the coverage file to the repo root"',
]
else:
if isinstance(test_command, str):
test_command = [Yaml.CodeBlock(test_command)]
# export UV_EXTRA_INDEX_URL="https://download.pytorch.org/whl/nightly/cpu https://download.pytorch.org/whl/nightly/cu126"
if self.config['use_pyproject_requirements']:
install_helpers = [
'echo "Installing helpers: setuptools"',
f'{self.PIP_INSTALL} --resolution=highest setuptools>=0.8 setuptools_scm wheel build -U', # is this necessary?
'echo "Installing helpers: tomli and pkginfo"',
f'{self.PIP_INSTALL} --resolution=highest tomli pkginfo packaging',
]
else:
install_helpers = [
'echo "Installing helpers: setuptools"',
f'{self.PIP_INSTALL} setuptools>=0.8 setuptools_scm wheel build -U', # is this necessary?
'echo "Installing helpers: tomli and pkginfo"',
f'{self.PIP_INSTALL} tomli pkginfo packaging',
]
# Note: export does not expose the environment variable to subsequent jobs.
install_wheel_commands = (
[
'echo "Finding the path to the wheel"',
f'ls {wheelhouse_dpath} || echo "{wheelhouse_dpath} does not exist"',
'echo "Installing helpers: update pip"',
f'{self.UPDATE_PIP}',
*install_helpers,
f'export WHEEL_FPATH=$({get_wheel_fpath_bash})',
# f'export MOD_VERSION=$({get_mod_version_bash})',
]
+ special_install_lines
+ [
'echo "WHEEL_FPATH=$WHEEL_FPATH"',
'echo "INSTALL_EXTRAS=$INSTALL_EXTRAS"',
'echo "UV_RESOLUTION=$UV_RESOLUTION"',
# 'echo "MOD_VERSION=$MOD_VERSION"',
# This helps but doesn't solve the problem.
# https://github.com/Erotemic/xdoctest/pull/158#discussion_r1697092781
# 'echo "Downloading dependencies from pypi"',
# f'pip download "{self.mod_name}[$INSTALL_EXTRAS]==$MOD_VERSION" --dest wheeldownload',
# f'echo "Overwriting pypi {self.mod_name} wheel"',
# 'cp wheelhouse/* wheeldownload/',
# f'pip install --prefer-binary "{self.mod_name}[$INSTALL_EXTRAS]==$MOD_VERSION" -f wheeldownload --no-index',
# TODO: flag to allow prerelease?
# f'{self.PIP_INSTALL_PREFER_BINARY} --prerelease=allow "{self.pkg_name}[$INSTALL_EXTRAS]==$MOD_VERSION" -f {wheelhouse_dpath}',
f'{self.PIP_INSTALL_PREFER_BINARY} "${{WHEEL_FPATH}}[${{INSTALL_EXTRAS}}]"',
'echo "Install finished."',
]
)
test_wheel_commands = (
[
'echo "Creating test sandbox directory"',
f'export WORKSPACE_DNAME="{workspace_dname}"',
'echo "WORKSPACE_DNAME=$WORKSPACE_DNAME"',
'mkdir -p $WORKSPACE_DNAME',
'echo "cd-ing into the workspace"',
'cd $WORKSPACE_DNAME',
'pwd',
'ls -altr',
# 'pip freeze',
'# Get the path to the installed package and run the tests',
f'export MOD_DPATH=$({get_modpath_bash})',
f'export MOD_NAME={self.mod_name}',
Yaml.CodeBlock(
"""
echo "
---
MOD_DPATH = $MOD_DPATH
---
running the pytest command inside the workspace
---
"
"""
),
]
+ custom_before_test_lines
+ test_command
+ custom_after_test_commands
)
install_and_test_wheel_parts = {
'install_wheel_commands': install_wheel_commands,
'test_wheel_commands': test_wheel_commands,
}
return install_and_test_wheel_parts