#!/usr/bin/env python
#
# utils.py
"""
General utilities.
"""
#
# Copyright © 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
# stdlib
import datetime
import os
import pathlib
import re
import textwrap
from datetime import date, timedelta
from io import StringIO
from types import ModuleType
from typing import (
Any,
Callable,
ContextManager,
Iterable,
Iterator,
List,
Mapping,
Optional,
TypeVar,
Union,
no_type_check
)
# 3rd party
import dulwich.repo
import isort
import isort.settings
import jinja2
import yapf_isort
from apeye.requests_url import RequestsURL
from domdf_python_tools.compat import importlib_resources
from domdf_python_tools.dates import calc_easter
from domdf_python_tools.import_tools import discover_entry_points
from domdf_python_tools.paths import PathPlus, sort_paths
from domdf_python_tools.pretty_print import FancyPrinter
from domdf_python_tools.stringlist import StringList
from domdf_python_tools.typing import PathLike
from jinja2 import Environment
from ruamel.yaml import YAML
from shippinglabel import normalize
from southwark import open_repo_closing, status
# this package
from repo_helper.configupdater2 import ConfigUpdater, Section
__all__ = [
"resource",
"IniConfigurator",
"discover_entry_points",
"easter_egg",
"indent_join",
"indent_with_tab",
"license_lookup",
"no_dev_versions",
"normalize",
"pformat_tabs",
"reformat_file",
"today",
"sort_paths",
"commit_changes",
"stage_changes",
"get_license_text",
"set_gh_actions_versions",
]
KT = TypeVar("KT")
VT = TypeVar("VT")
#: Under normal circumstances returns :meth:`datetime.date.today`.
today: date = date.today()
[docs]def indent_with_tab(
text: str,
depth: int = 1,
predicate: Optional[Callable[[str], bool]] = None,
) -> str:
r"""
Adds ``'\t'`` to the beginning of selected lines in 'text'.
:param text: The text to indent.
:param depth: The depth of the indentation.
:param predicate: If given, ``'\t'`` will only be added to the lines where ``predicate(line)``
is :py:obj`True`. If ``predicate`` is not provided, it will default to adding ``'\t'``
to all non-empty lines that do not consist solely of whitespace characters.
"""
return textwrap.indent(text, '\t' * depth, predicate=predicate)
#: Mapping of license short codes to license names used in trove classifiers.
license_lookup = {
"AFL-1.1": "Academic Free License (AFL)",
"AFL-1.2": "Academic Free License (AFL)",
"AFL-2.0": "Academic Free License (AFL)",
"AFL-2.1": "Academic Free License (AFL)",
"AFL-3.0": "Academic Free License (AFL)",
"Apache": "Apache Software License",
"Apache-1.0": "Apache Software License",
"Apache-1.1": "Apache Software License",
"Apache-2.0": "Apache Software License",
"APSL-1.0": "Apple Public Source License",
"APSL-1.1": "Apple Public Source License",
"APSL-1.2": "Apple Public Source License",
"APSL-2.0": "Apple Public Source License",
"Artistic-1.0": "Artistic License",
"AAL": "Attribution Assurance License",
"BSD": "BSD License",
"BSD-2-Clause": "BSD License",
"BSD-3-Clause": "BSD License",
"BSL-1.0": "Boost Software License 1.0 (BSL-1.0)",
"CDDL-1.0": "Common Development and Distribution License 1.0 (CDDL-1.0)",
"CPL-1.0": "Common Public License",
"EPL-1.0": "Eclipse Public License 1.0 (EPL-1.0)",
"EPL-2.0": "Eclipse Public License 2.0 (EPL-2.0)",
"EFL-1.0": "Eiffel Forum License",
"EFL-2.0": "Eiffel Forum License",
"EUPL 1.0": "European Union Public Licence 1.0 (EUPL 1.0)",
"EUPL 1.1": "European Union Public Licence 1.1 (EUPL 1.1)",
"EUPL 1.2": "European Union Public Licence 1.2 (EUPL 1.2)",
"AGPL-3.0-only": "GNU Affero General Public License v3",
"AGPL-3.0": "GNU Affero General Public License v3",
"AGPL-3.0-or-later": "GNU Affero General Public License v3 or later (AGPLv3+)",
"AGPL-3.0+": "GNU Affero General Public License v3 or later (AGPLv3+)",
"FDL": "GNU Free Documentation License (FDL)",
"GFDL-1.1-only": "GNU Free Documentation License (FDL)",
"GFDL-1.1-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.2-only": "GNU Free Documentation License (FDL)",
"GFDL-1.2-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.3-only": "GNU Free Documentation License (FDL)",
"GFDL-1.3-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.2": "GNU Free Documentation License (FDL)",
"GFDL-1.1": "GNU Free Documentation License (FDL)",
"GFDL-1.3": "GNU Free Documentation License (FDL)",
"GPL": "GNU General Public License (GPL)",
"GPL-1.0-only": "GNU General Public License (GPL)",
"GPL-1.0-or-later": "GNU General Public License (GPL)",
"GPLv2": "GNU General Public License v2 (GPLv2)",
"GPL-2.0-only": "GNU General Public License v2 (GPLv2)",
"GPLv2+": "GNU General Public License v2 or later (GPLv2+)",
"GPL-2.0-or-later": "GNU General Public License v2 or later (GPLv2+)",
"GPLv3": "GNU General Public License v3 (GPLv3)",
"GPL-3.0-only": "GNU General Public License v3 (GPLv3)",
"GPLv3+": "GNU General Public License v3 or later (GPLv3+)",
"GPL-3.0-or-later": "GNU General Public License v3 or later (GPLv3+)",
"LGPLv2": "GNU Lesser General Public License v2 (LGPLv2)",
"LGPLv2+": "GNU Lesser General Public License v2 or later (LGPLv2+)",
"LGPLv3": "GNU Lesser General Public License v3 (LGPLv3)",
"LGPL-3.0-only": "GNU Lesser General Public License v3 (LGPLv3)",
"LGPLv3+": "GNU Lesser General Public License v3 or later (LGPLv3+)",
"LGPL-3.0-or-later": "GNU Lesser General Public License v3 or later (LGPLv3+)",
"LGPL": "GNU Library or Lesser General Public License (LGPL)",
"HPND": "Historical Permission Notice and Disclaimer (HPND)",
"IPL-1.0": "IBM Public License",
"ISCL": "ISC License (ISCL)",
"Intel": "Intel Open Source License",
"MIT": "MIT License",
"MirOS": "MirOS License (MirOS)",
"Motosoto": "Motosoto License",
"MPL": "Mozilla Public License 1.0 (MPL)",
"MPL-1.0": "Mozilla Public License 1.0 (MPL)",
"MPL 1.1": "Mozilla Public License 1.1 (MPL 1.1)",
"MPL 2.0": "Mozilla Public License 2.0 (MPL 2.0)",
"NGPL": "Nethack General Public License",
"Nokia": "Nokia Open Source License",
"OGTSL": "Open Group Test Suite License",
"OSL-3.0": "Open Software License 3.0 (OSL-3.0)",
"PostgreSQL": "PostgreSQL License",
"CNRI-Python": "Python License (CNRI Python License)",
"PSF-2.0": "Python Software Foundation License",
"QPL-1.0": "Qt Public License (QPL)",
"RSCPL": "Ricoh Source Code Public License",
"OFL-1.1": "SIL Open Font License 1.1 (OFL-1.1)",
"Sleepycat": "Sleepycat License",
"SISSL": "Sun Industry Standards Source License (SISSL)",
"SISSL-1.2": "Sun Industry Standards Source License (SISSL)",
"SPL-1.0": "Sun Public License",
"UPL": "Universal Permissive License (UPL)",
"UPL-1.0": "Universal Permissive License (UPL)",
"NCSA": "University of Illinois/NCSA Open Source License",
"VSL-1.0": "Vovida Software License 1.0",
"W3C": "W3C License",
"Xnet": "X.Net License",
"ZPL-1.1": "Zope Public License",
"ZPL-2.0": "Zope Public License",
"ZPL-2.1": "Zope Public License",
"Zlib": "zlib/libpng License",
"Proprietary": "Other/Proprietary License",
"Other": "Other/Proprietary License",
"PD": "Public Domain",
"Public Domain": "Public Domain",
}
_reverse_license_lookup = {v: k for k, v in reversed(list(license_lookup.items()))}
[docs]def indent_join(iterable: Iterable[str]) -> str:
"""
Join an iterable of strings with newlines,
and indent each line with a tab if there is more then one element.
:param iterable:
""" # noqa: D400
iterable = list(iterable)
if len(iterable) > 1:
if not iterable[0] == '':
iterable.insert(0, '')
return indent_with_tab(textwrap.dedent('\n'.join(iterable)))
[docs]class IniConfigurator:
"""
Base class to generate ``.ini`` configuration files.
:param base_path:
"""
managed_sections: List[str]
_ini: ConfigUpdater
_output: StringList
managed_message: str = "This file is managed by 'repo_helper'."
filename: str
def __init__(self, base_path: pathlib.Path):
self.base_path = base_path
self._ini = ConfigUpdater()
self._output = StringList([
f"# {self.managed_message}",
"# You may add new sections, but any changes made to the following sections will be lost:",
])
self.managed_sections = self.managed_sections[:]
for sec in self.managed_sections:
self._ini.add_section(sec)
self._output.append(f"# * {sec}")
self._output.blankline(ensure_single=True)
[docs] def merge_existing(self, ini_file: pathlib.Path) -> None:
"""
Merge existing sections in the configuration file into the new configuration.
:param ini_file: The existing ``.ini`` file.
"""
if ini_file.is_file():
existing_config = ConfigUpdater()
existing_config.read(str(ini_file))
for section in existing_config.sections_blocks():
if section.name not in self.managed_sections:
self._ini.add_section(section)
[docs] def write_out(self) -> None:
"""
Write out to the ``.ini`` file.
"""
ini_file = PathPlus(self.base_path / self.filename)
for section_name in self.managed_sections:
getattr(self, re.sub("[:.-]", '_', section_name))()
self.merge_existing(ini_file)
self._output.append(str(self._ini))
ini_file.write_lines(self._output)
[docs] def copy_existing_value(self, section: Section, key: str) -> None:
"""
Copy the existing value for ``key``, if present, to the new configuration.
:param section:
:param key:
"""
if key in section:
self._ini[section.name][key] = section[key].value
def easter_egg() -> None: # noqa: D103 # pragma: no cover
easter = calc_easter(today.year)
easter_margin = timedelta(days=7)
if today - easter_margin <= easter <= today + easter_margin:
print("🐇 🐣 🥚")
elif date(today.year, 10, 24) <= today <= date(today.year, 11, 2):
print("🎃 👻 🦇")
elif today == date(today.year, 11, 5):
print("🎆 🔥 🚀")
elif today == date(today.year, 11, 11):
print("We will remember them.")
elif today.month == 12:
print("🎅 ☃️ 🎁")
[docs]def no_dev_versions(versions: Iterable[str]) -> List[str]:
"""
Returns the subset of ``versions`` which does not end with ``-dev``.
:param versions:
"""
return [v for v in versions if not v.endswith("-dev")]
[docs]def stage_changes(
repo: Union[PathLike, dulwich.repo.Repo],
files: Iterable[PathLike],
) -> List[PathPlus]:
"""
Stage any files that have been updated, added or removed.
:param repo: The repository.
:param files: List of files to stage.
:returns: A list of staged files.
Not all files in ``files`` will have been changed, and only changes are staged.
.. versionadded:: 2020.11.23
"""
with open_repo_closing(repo) as repo:
stat = status(repo)
unstaged_changes = stat.unstaged
untracked_files = stat.untracked
staged_files = []
for filename in files:
filename = PathPlus(filename)
if filename.is_absolute():
filename = filename.relative_to(repo.path)
if filename in unstaged_changes or filename in untracked_files:
repo.stage(os.path.normpath(filename))
staged_files.append(filename)
elif (
filename in stat.staged["add"] or filename in stat.staged["modify"]
or filename in stat.staged["delete"]
):
staged_files.append(filename)
return staged_files
[docs]def commit_changes(
repo: Union[PathLike, dulwich.repo.Repo],
message: str = "Updated files with 'repo_helper'.",
) -> str:
"""
Commit staged changes.
:param repo: The repository to commit in.
:param message: The commit message to use.
:returns: The SHA of the commit.
.. versionadded:: 2020.11.23
"""
with open_repo_closing(repo) as repo:
current_time = datetime.datetime.now(datetime.timezone.utc).astimezone()
tzinfo = current_time.tzinfo
assert tzinfo is not None
time_offset = tzinfo.utcoffset(None)
assert time_offset is not None
current_timezone = time_offset.total_seconds()
commit_sha = repo.do_commit(
message=message.encode("UTF-8"),
commit_timestamp=current_time.timestamp(),
commit_timezone=current_timezone,
)
return commit_sha.decode("UTF-8")
def brace(string: str) -> str:
return f"{{{{ {string} }}}}"
[docs]def set_gh_actions_versions(py_versions: Iterable[str]) -> List[str]:
"""
Convert development Python versions into the appropriate versions for GitHub Actions.
:param py_versions:
"""
py_versions = list(py_versions)
# Keep in sync with https://github.com/actions/python-versions/releases
if "3.9-dev" in py_versions:
py_versions[py_versions.index("3.9-dev")] = "3.9"
if "3.10-dev" in py_versions:
py_versions[py_versions.index("3.10-dev")] = "3.10"
if "3.11-dev" in py_versions:
py_versions[py_versions.index("3.11-dev")] = "3.11"
if "3.12-dev" in py_versions:
py_versions[py_versions.index("3.12-dev")] = "3.12"
if "3.13-dev" in py_versions:
py_versions[py_versions.index("3.13-dev")] = "3.13"
if "3.14-dev" in py_versions:
py_versions[py_versions.index("3.14-dev")] = "3.14"
if "3.15-dev" in py_versions:
py_versions[py_versions.index("3.15-dev")] = "3.15.0-alpha.3"
if "3.15" in py_versions:
py_versions[py_versions.index("3.15")] = "3.15.0-alpha.3"
if "pypy3" in py_versions:
py_versions[py_versions.index("pypy3")] = "pypy-3.6"
if "pypy36" in py_versions:
py_versions[py_versions.index("pypy36")] = "pypy-3.6"
if "pypy3.6" in py_versions:
py_versions[py_versions.index("pypy3.6")] = "pypy-3.6"
if "pypy37" in py_versions:
py_versions[py_versions.index("pypy37")] = "pypy-3.7"
if "pypy3.7" in py_versions:
py_versions[py_versions.index("pypy3.7")] = "pypy-3.7"
if "pypy38" in py_versions:
py_versions[py_versions.index("pypy38")] = "pypy-3.8"
if "pypy3.8" in py_versions:
py_versions[py_versions.index("pypy3.8")] = "pypy-3.8"
if "pypy39" in py_versions:
py_versions[py_versions.index("pypy39")] = "pypy-3.9"
if "pypy3.9" in py_versions:
py_versions[py_versions.index("pypy3.9")] = "pypy-3.9"
if "pypy310" in py_versions:
py_versions[py_versions.index("pypy310")] = "pypy-3.10"
if "pypy3.10" in py_versions:
py_versions[py_versions.index("pypy3.10")] = "pypy-3.10"
if "rustpython" in py_versions:
py_versions.remove("rustpython")
return py_versions
_yaml_round_trip_dumper = YAML(typ="rt")
_yaml_round_trip_dumper.default_flow_style = False
def _round_trip_dump(obj: Any) -> str:
stream = StringIO()
_yaml_round_trip_dumper.dump(obj, stream=stream)
return stream.getvalue()
[docs]@no_type_check
def resource(
package: Union[str, ModuleType],
resource: PathLike,
) -> ContextManager[pathlib.Path]:
"""
Retrieve the path to a resource inside a package.
.. versionadded:: 2022.4.4
:param package: The name of the package, or a module object representing it.
:param resource: The name of the resource.
"""
return importlib_resources.as_file(importlib_resources.files(package) / os.fspath(resource))
base_license_url = RequestsURL("https://raw.githubusercontent.com/licenses/license-templates/master/templates/")
license_file_lookup = dict([
(
"GNU Lesser General Public License v3 (LGPLv3)",
(base_license_url / "lgpl.txt", "lgpl3.py"),
),
(
"GNU Lesser General Public License v3 or later (LGPLv3+)",
(base_license_url / "lgpl.txt", "lgpl3_plus.py"),
),
("GNU General Public License v3 (GPLv3)", (base_license_url / "gpl3.txt", "gpl3.py")),
("GNU General Public License v3 or later (GPLv3+)", (base_license_url / "gpl3.txt", "gpl3_plus.py")),
(
"GNU General Public License v2 (GPLv2)",
(RequestsURL("https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"), "gpl2.py"),
),
(
"GNU General Public License v2 or later (GPLv2+)",
(RequestsURL("https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"), "gpl2_plus.py"),
),
("MIT License", (base_license_url / "mit.txt", "mit.py")),
])
[docs]def get_license_text(
license_name: str,
copyright_years: Union[str, int],
author: str,
project_name: str,
) -> str:
"""
Obtain the license text for the given license.
.. versionadded:: 2022.4.4
:param license_name: The name of the license.
:param copyright_years: The copyright years (e.g. ``'2019-2021'``) to display in the license.
Not supported by all licenses.
:param author: The name of the author/copyright holder to display in the license.
Not supported by all licenses.
:param project_name: The name of the project to display in the license.
Not supported by all licenses.
Licenses are obtained from https://github.com/licenses/license-templates.
"""
if license_name in license_lookup:
license_name = license_lookup[license_name]
# Licenses from https://github.com/licenses/license-templates/tree/master/templates
license_url: Optional[RequestsURL] = None
license_text: str = ''
# TODO: 2 vs 3 clause BSD
if license_name in license_file_lookup:
license_url = license_file_lookup[license_name][0]
elif license_name == "BSD License":
license_url = base_license_url / "bsd2.txt"
elif license_name == "Apache Software License":
license_url = base_license_url / "apache.txt"
if license_url is not None:
for attempt in [1, 2]:
try:
response = license_url.get()
except Exception:
# except requests.exceptions.RequestException:
if attempt == 1:
continue
else:
raise
if response.status_code == 200:
license_text = response.text
license_template = Environment(
loader=jinja2.BaseLoader(),
undefined=jinja2.StrictUndefined,
).from_string(license_text)
return license_template.render(
year=copyright_years,
organization=author,
project=project_name,
)
def get_keys(mapping: Mapping[KT, VT], *keys: KT) -> Iterator[VT]:
r"""
Returns an iterator over ``\*keys`` from ``mapping``.
:param mapping:
:param \*keys:
"""
for key in keys:
yield mapping[key]