#!/usr/bin/env python
#
# utils.py
"""
General utilities.
"""
#
# Copyright © 2020 Dominic Davis-Foster <dominic@davis-foster.co.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
# stdlib
import datetime
import os
import pathlib
import re
import sys
import textwrap
from datetime import date, timedelta
from io import StringIO
from types import ModuleType
from typing import Any, Callable, Iterable, Iterator, List, Mapping, Optional, TypeVar, Union, no_type_check
# 3rd party
import dulwich.repo
import isort
import isort.settings
import jinja2
import yapf_isort
from apeye.requests_url import RequestsURL
from domdf_python_tools.compat import importlib_resources
from domdf_python_tools.dates import calc_easter
from domdf_python_tools.import_tools import discover_entry_points
from domdf_python_tools.paths import PathPlus, sort_paths
from domdf_python_tools.pretty_print import FancyPrinter
from domdf_python_tools.stringlist import StringList
from domdf_python_tools.typing import PathLike
from jinja2 import Environment
from ruamel.yaml import YAML
from shippinglabel import normalize
from southwark import open_repo_closing, status
from typing_extensions import ContextManager
# this package
from repo_helper.configupdater2 import ConfigUpdater, Section
__all__ = [
"resource",
"IniConfigurator",
"discover_entry_points",
"easter_egg",
"indent_join",
"indent_with_tab",
"license_lookup",
"no_dev_versions",
"normalize",
"pformat_tabs",
"reformat_file",
"today",
"sort_paths",
"commit_changes",
"stage_changes",
"get_license_text",
"set_gh_actions_versions",
]
KT = TypeVar("KT")
VT = TypeVar("VT")
#: Under normal circumstances returns :meth:`datetime.date.today`.
today: date = date.today()
[docs]def indent_with_tab(
text: str,
depth: int = 1,
predicate: Optional[Callable[[str], bool]] = None,
) -> str:
r"""
Adds ``'\t'`` to the beginning of selected lines in 'text'.
:param text: The text to indent.
:param depth: The depth of the indentation.
:param predicate: If given, ``'\t'`` will only be added to the lines where ``predicate(line)``
is :py:obj`True`. If ``predicate`` is not provided, it will default to adding ``'\t'``
to all non-empty lines that do not consist solely of whitespace characters.
"""
return textwrap.indent(text, '\t' * depth, predicate=predicate)
#: Mapping of license short codes to license names used in trove classifiers.
license_lookup = {
"AFL-1.1": "Academic Free License (AFL)",
"AFL-1.2": "Academic Free License (AFL)",
"AFL-2.0": "Academic Free License (AFL)",
"AFL-2.1": "Academic Free License (AFL)",
"AFL-3.0": "Academic Free License (AFL)",
"Apache": "Apache Software License",
"Apache-1.0": "Apache Software License",
"Apache-1.1": "Apache Software License",
"Apache-2.0": "Apache Software License",
"APSL-1.0": "Apple Public Source License",
"APSL-1.1": "Apple Public Source License",
"APSL-1.2": "Apple Public Source License",
"APSL-2.0": "Apple Public Source License",
"Artistic-1.0": "Artistic License",
"AAL": "Attribution Assurance License",
"BSD": "BSD License",
"BSD-2-Clause": "BSD License",
"BSD-3-Clause": "BSD License",
"BSL-1.0": "Boost Software License 1.0 (BSL-1.0)",
"CDDL-1.0": "Common Development and Distribution License 1.0 (CDDL-1.0)",
"CPL-1.0": "Common Public License",
"EPL-1.0": "Eclipse Public License 1.0 (EPL-1.0)",
"EPL-2.0": "Eclipse Public License 2.0 (EPL-2.0)",
"EFL-1.0": "Eiffel Forum License",
"EFL-2.0": "Eiffel Forum License",
"EUPL 1.0": "European Union Public Licence 1.0 (EUPL 1.0)",
"EUPL 1.1": "European Union Public Licence 1.1 (EUPL 1.1)",
"EUPL 1.2": "European Union Public Licence 1.2 (EUPL 1.2)",
"AGPL-3.0-only": "GNU Affero General Public License v3",
"AGPL-3.0": "GNU Affero General Public License v3",
"AGPL-3.0-or-later": "GNU Affero General Public License v3 or later (AGPLv3+)",
"AGPL-3.0+": "GNU Affero General Public License v3 or later (AGPLv3+)",
"FDL": "GNU Free Documentation License (FDL)",
"GFDL-1.1-only": "GNU Free Documentation License (FDL)",
"GFDL-1.1-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.2-only": "GNU Free Documentation License (FDL)",
"GFDL-1.2-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.3-only": "GNU Free Documentation License (FDL)",
"GFDL-1.3-or-later": "GNU Free Documentation License (FDL)",
"GFDL-1.2": "GNU Free Documentation License (FDL)",
"GFDL-1.1": "GNU Free Documentation License (FDL)",
"GFDL-1.3": "GNU Free Documentation License (FDL)",
"GPL": "GNU General Public License (GPL)",
"GPL-1.0-only": "GNU General Public License (GPL)",
"GPL-1.0-or-later": "GNU General Public License (GPL)",
"GPLv2": "GNU General Public License v2 (GPLv2)",
"GPL-2.0-only": "GNU General Public License v2 (GPLv2)",
"GPLv2+": "GNU General Public License v2 or later (GPLv2+)",
"GPL-2.0-or-later": "GNU General Public License v2 or later (GPLv2+)",
"GPLv3": "GNU General Public License v3 (GPLv3)",
"GPL-3.0-only": "GNU General Public License v3 (GPLv3)",
"GPLv3+": "GNU General Public License v3 or later (GPLv3+)",
"GPL-3.0-or-later": "GNU General Public License v3 or later (GPLv3+)",
"LGPLv2": "GNU Lesser General Public License v2 (LGPLv2)",
"LGPLv2+": "GNU Lesser General Public License v2 or later (LGPLv2+)",
"LGPLv3": "GNU Lesser General Public License v3 (LGPLv3)",
"LGPL-3.0-only": "GNU Lesser General Public License v3 (LGPLv3)",
"LGPLv3+": "GNU Lesser General Public License v3 or later (LGPLv3+)",
"LGPL-3.0-or-later": "GNU Lesser General Public License v3 or later (LGPLv3+)",
"LGPL": "GNU Library or Lesser General Public License (LGPL)",
"HPND": "Historical Permission Notice and Disclaimer (HPND)",
"IPL-1.0": "IBM Public License",
"ISCL": "ISC License (ISCL)",
"Intel": "Intel Open Source License",
"MIT": "MIT License",
"MirOS": "MirOS License (MirOS)",
"Motosoto": "Motosoto License",
"MPL": "Mozilla Public License 1.0 (MPL)",
"MPL-1.0": "Mozilla Public License 1.0 (MPL)",
"MPL 1.1": "Mozilla Public License 1.1 (MPL 1.1)",
"MPL 2.0": "Mozilla Public License 2.0 (MPL 2.0)",
"NGPL": "Nethack General Public License",
"Nokia": "Nokia Open Source License",
"OGTSL": "Open Group Test Suite License",
"OSL-3.0": "Open Software License 3.0 (OSL-3.0)",
"PostgreSQL": "PostgreSQL License",
"CNRI-Python": "Python License (CNRI Python License)",
"PSF-2.0": "Python Software Foundation License",
"QPL-1.0": "Qt Public License (QPL)",
"RSCPL": "Ricoh Source Code Public License",
"OFL-1.1": "SIL Open Font License 1.1 (OFL-1.1)",
"Sleepycat": "Sleepycat License",
"SISSL": "Sun Industry Standards Source License (SISSL)",
"SISSL-1.2": "Sun Industry Standards Source License (SISSL)",
"SPL-1.0": "Sun Public License",
"UPL": "Universal Permissive License (UPL)",
"UPL-1.0": "Universal Permissive License (UPL)",
"NCSA": "University of Illinois/NCSA Open Source License",
"VSL-1.0": "Vovida Software License 1.0",
"W3C": "W3C License",
"Xnet": "X.Net License",
"ZPL-1.1": "Zope Public License",
"ZPL-2.0": "Zope Public License",
"ZPL-2.1": "Zope Public License",
"Zlib": "zlib/libpng License",
"Proprietary": "Other/Proprietary License",
"Other": "Other/Proprietary License",
"PD": "Public Domain",
"Public Domain": "Public Domain",
}
[docs]def indent_join(iterable: Iterable[str]) -> str:
"""
Join an iterable of strings with newlines,
and indent each line with a tab if there is more then one element.
:param iterable:
""" # noqa: D400
iterable = list(iterable)
if len(iterable) > 1:
if not iterable[0] == '':
iterable.insert(0, '')
return indent_with_tab(textwrap.dedent('\n'.join(iterable)))
[docs]class IniConfigurator:
"""
Base class to generate ``.ini`` configuration files.
:param base_path:
"""
managed_sections: List[str]
_ini: ConfigUpdater
_output: StringList
managed_message: str = "This file is managed by 'repo_helper'."
filename: str
def __init__(self, base_path: pathlib.Path):
self.base_path = base_path
self._ini = ConfigUpdater()
self._output = StringList([
f"# {self.managed_message}",
"# You may add new sections, but any changes made to the following sections will be lost:",
])
self.managed_sections = self.managed_sections[:]
for sec in self.managed_sections:
self._ini.add_section(sec)
self._output.append(f"# * {sec}")
self._output.blankline(ensure_single=True)
[docs] def merge_existing(self, ini_file: pathlib.Path) -> None:
"""
Merge existing sections in the configuration file into the new configuration.
:param ini_file: The existing ``.ini`` file.
"""
if ini_file.is_file():
existing_config = ConfigUpdater()
existing_config.read(str(ini_file))
for section in existing_config.sections_blocks():
if section.name not in self.managed_sections:
self._ini.add_section(section)
[docs] def write_out(self) -> None:
"""
Write out to the ``.ini`` file.
"""
ini_file = PathPlus(self.base_path / self.filename)
for section_name in self.managed_sections:
getattr(self, re.sub("[:.-]", '_', section_name))()
self.merge_existing(ini_file)
self._output.append(str(self._ini))
ini_file.write_lines(self._output)
[docs] def copy_existing_value(self, section: Section, key: str) -> None:
"""
Copy the existing value for ``key``, if present, to the new configuration.
:param section:
:param key:
"""
if key in section:
self._ini[section.name][key] = section[key].value
def easter_egg() -> None: # noqa: D103 # pragma: no cover
easter = calc_easter(today.year)
easter_margin = timedelta(days=7)
if today - easter_margin <= easter <= today + easter_margin:
print("🐇 🐣 🥚")
elif date(today.year, 10, 24) <= today <= date(today.year, 11, 2):
print("🎃 👻 🦇")
elif today == date(today.year, 11, 5):
print("🎆 🔥 🚀")
elif today == date(today.year, 11, 11):
print("We will remember them.")
elif today.month == 12:
print("🎅 ☃️ 🎁")
[docs]def no_dev_versions(versions: Iterable[str]) -> List[str]:
"""
Returns the subset of ``versions`` which does not end with ``-dev``.
:param versions:
"""
return [v for v in versions if not v.endswith("-dev")]
[docs]def stage_changes(
repo: Union[PathLike, dulwich.repo.Repo],
files: Iterable[PathLike],
) -> List[PathPlus]:
"""
Stage any files that have been updated, added or removed.
:param repo: The repository.
:param files: List of files to stage.
:returns: A list of staged files.
Not all files in ``files`` will have been changed, and only changes are staged.
.. versionadded:: 2020.11.23
"""
with open_repo_closing(repo) as repo:
stat = status(repo)
unstaged_changes = stat.unstaged
untracked_files = stat.untracked
staged_files = []
for filename in files:
filename = PathPlus(filename)
if filename.is_absolute():
filename = filename.relative_to(repo.path)
if filename in unstaged_changes or filename in untracked_files:
repo.stage(os.path.normpath(filename))
staged_files.append(filename)
elif (
filename in stat.staged["add"] or filename in stat.staged["modify"]
or filename in stat.staged["delete"]
):
staged_files.append(filename)
return staged_files
[docs]def commit_changes(
repo: Union[PathLike, dulwich.repo.Repo],
message: str = "Updated files with 'repo_helper'.",
) -> str:
"""
Commit staged changes.
:param repo: The repository to commit in.
:param message: The commit message to use.
:returns: The SHA of the commit.
.. versionadded:: 2020.11.23
"""
with open_repo_closing(repo) as repo:
current_time = datetime.datetime.now(datetime.timezone.utc).astimezone()
tzinfo = current_time.tzinfo
assert tzinfo is not None
time_offset = tzinfo.utcoffset(None)
assert time_offset is not None
current_timezone = time_offset.total_seconds()
commit_sha = repo.do_commit(
message=message.encode("UTF-8"),
commit_timestamp=current_time.timestamp(),
commit_timezone=current_timezone,
)
return commit_sha.decode("UTF-8")
def brace(string: str) -> str:
return f"{{{{ {string} }}}}"
[docs]def set_gh_actions_versions(py_versions: Iterable[str]) -> List[str]:
"""
Convert development Python versions into the appropriate versions for GitHub Actions.
:param py_versions:
"""
py_versions = list(py_versions)
# Keep in sync with https://github.com/actions/python-versions/releases
if "3.9-dev" in py_versions:
py_versions[py_versions.index("3.9-dev")] = "3.9"
if "3.10-dev" in py_versions:
py_versions[py_versions.index("3.10-dev")] = "3.10"
if "3.11-dev" in py_versions:
py_versions[py_versions.index("3.11-dev")] = "3.11"
if "3.12-dev" in py_versions:
py_versions[py_versions.index("3.12-dev")] = "3.12"
if "3.13-dev" in py_versions:
py_versions[py_versions.index("3.13-dev")] = "3.13.0-alpha.6"
if "3.13" in py_versions:
py_versions[py_versions.index("3.13")] = "3.13.0-alpha.6"
if "pypy3" in py_versions:
py_versions[py_versions.index("pypy3")] = "pypy-3.6"
if "pypy36" in py_versions:
py_versions[py_versions.index("pypy36")] = "pypy-3.6"
if "pypy3.6" in py_versions:
py_versions[py_versions.index("pypy3.6")] = "pypy-3.6"
if "pypy37" in py_versions:
py_versions[py_versions.index("pypy37")] = "pypy-3.7"
if "pypy3.7" in py_versions:
py_versions[py_versions.index("pypy3.7")] = "pypy-3.7"
if "pypy38" in py_versions:
py_versions[py_versions.index("pypy38")] = "pypy-3.8"
if "pypy3.8" in py_versions:
py_versions[py_versions.index("pypy3.8")] = "pypy-3.8"
if "pypy39" in py_versions:
py_versions[py_versions.index("pypy39")] = "pypy-3.9"
if "pypy3.9" in py_versions:
py_versions[py_versions.index("pypy3.9")] = "pypy-3.9"
if "pypy310" in py_versions:
py_versions[py_versions.index("pypy310")] = "pypy-3.10"
if "pypy3.10" in py_versions:
py_versions[py_versions.index("pypy3.10")] = "pypy-3.10"
if "rustpython" in py_versions:
py_versions.remove("rustpython")
return py_versions
_yaml_round_trip_dumper = YAML(typ="rt")
_yaml_round_trip_dumper.default_flow_style = False
def _round_trip_dump(obj: Any) -> str:
stream = StringIO()
_yaml_round_trip_dumper.dump(obj, stream=stream)
return stream.getvalue()
[docs]@no_type_check
def resource(
package: Union[str, ModuleType],
resource: PathLike,
) -> ContextManager[pathlib.Path]:
"""
Retrieve the path to a resource inside a package.
.. versionadded:: 2022.4.4
:param package: The name of the package, or a module object representing it.
:param resource: The name of the resource.
"""
if sys.version_info < (3, 7) or sys.version_info >= (3, 11):
return importlib_resources.as_file(importlib_resources.files(package) / os.fspath(resource))
else:
return importlib_resources.path(package, resource)
base_license_url = RequestsURL("https://raw.githubusercontent.com/licenses/license-templates/master/templates/")
license_file_lookup = dict([
(
"GNU Lesser General Public License v3 (LGPLv3)",
(base_license_url / "lgpl.txt", "lgpl3.py"),
),
(
"GNU Lesser General Public License v3 or later (LGPLv3+)",
(base_license_url / "lgpl.txt", "lgpl3_plus.py")
),
("GNU General Public License v3 (GPLv3)", (base_license_url / "gpl3.txt", "gpl3.py")),
("GNU General Public License v3 or later (GPLv3+)", (base_license_url / "gpl3.txt", "gpl3_plus.py")),
(
"GNU General Public License v2 (GPLv2)",
(RequestsURL("https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"), "gpl2.py"),
),
(
"GNU General Public License v2 or later (GPLv2+)",
(RequestsURL("https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"), "gpl2_plus.py")
),
("MIT License", (base_license_url / "mit.txt", "mit.py")),
])
[docs]def get_license_text(
license_name: str,
copyright_years: Union[str, int],
author: str,
project_name: str,
) -> str:
"""
Obtain the license text for the given license.
.. versionadded:: 2022.4.4
:param license_name: The name of the license.
:param copyright_years: The copyright years (e.g. ``'2019-2021'``) to display in the license.
Not supported by all licenses.
:param author: The name of the author/copyright holder to display in the license.
Not supported by all licenses.
:param project_name: The name of the project to display in the license.
Not supported by all licenses.
Licenses are obtained from https://github.com/licenses/license-templates.
"""
if license_name in license_lookup:
license_name = license_lookup[license_name]
# Licenses from https://github.com/licenses/license-templates/tree/master/templates
license_url: Optional[RequestsURL] = None
license_text: str = ''
# TODO: 2 vs 3 clause BSD
if license_name in license_file_lookup:
license_url = license_file_lookup[license_name][0]
elif license_name == "BSD License":
license_url = base_license_url / "bsd2.txt"
elif license_name == "Apache Software License":
license_url = base_license_url / "apache.txt"
if license_url is not None:
for attempt in [1, 2]:
try:
response = license_url.get()
except Exception:
# except requests.exceptions.RequestException:
if attempt == 1:
continue
else:
raise
if response.status_code == 200:
license_text = response.text
license_template = Environment(
loader=jinja2.BaseLoader(),
undefined=jinja2.StrictUndefined,
).from_string(license_text)
return license_template.render(
year=copyright_years,
organization=author,
project=project_name,
)
def get_keys(mapping: Mapping[KT, VT], *keys: KT) -> Iterator[VT]:
r"""
Returns an iterator over ``\*keys`` from ``mapping``.
:param mapping:
:param \*keys:
"""
for key in keys:
yield mapping[key]