Skip to content
Snippets Groups Projects
Commit 4d4735e8f621 authored by Jason R. Coombs's avatar Jason R. Coombs
Browse files

bpo-42382: In importlib.metadata, `EntryPoint` objects now expose `dist` (#23758)

* bpo-42382: In importlib.metadata, `EntryPoint` objects now expose a `.dist` object referencing the `Distribution` when constructed from a `Distribution`.

Also, sync importlib_metadata 3.3:

- Add support for package discovery under package normalization rules.
- The object returned by `metadata()` now has a formally-defined protocol called `PackageMetadata` with declared support for the `.get_all()` method.

* Add blurb

* Remove latent footnote.
parent 0326c713a0c7
Branches
No related tags found
No related merge requests found
......@@ -115,8 +115,9 @@
>>> wheel_metadata = metadata('wheel') # doctest: +SKIP
The keys of the returned data structure [#f1]_ name the metadata keywords, and
their values are returned unparsed from the distribution metadata::
The keys of the returned data structure, a ``PackageMetadata``,
name the metadata keywords, and
the values are returned unparsed from the distribution metadata::
>>> wheel_metadata['Requires-Python'] # doctest: +SKIP
'>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*'
......@@ -259,9 +260,3 @@
.. rubric:: Footnotes
.. [#f1] Technically, the returned distribution metadata object is an
:class:`email.message.EmailMessage`
instance, but this is an implementation detail, and not part of the
stable API. You should only use dictionary-like methods and syntax
to access the metadata contents.
import io
import os
import re
import abc
......@@ -18,6 +17,7 @@
from importlib import import_module
from importlib.abc import MetaPathFinder
from itertools import starmap
from typing import Any, List, Optional, Protocol, TypeVar, Union
__all__ = [
......@@ -43,7 +43,7 @@
@property
def name(self):
name, = self.args
(name,) = self.args
return name
......@@ -77,6 +77,8 @@
following the attr, and following any extras.
"""
dist: Optional['Distribution'] = None
def load(self):
"""Load the entry point from its definition. If only a module
is indicated by the value, return that module. Otherwise,
......@@ -104,7 +106,7 @@
@classmethod
def _from_config(cls, config):
return [
return (
cls(name, value, group)
for group in config.sections()
for name, value in config.items(group)
......@@ -108,10 +110,10 @@
cls(name, value, group)
for group in config.sections()
for name, value in config.items(group)
]
)
@classmethod
def _from_text(cls, text):
config = ConfigParser(delimiters='=')
# case sensitive: https://stackoverflow.com/q/1611799/812183
config.optionxform = str
......@@ -112,8 +114,7 @@
@classmethod
def _from_text(cls, text):
config = ConfigParser(delimiters='=')
# case sensitive: https://stackoverflow.com/q/1611799/812183
config.optionxform = str
try:
config.read_string(text)
......@@ -119,8 +120,13 @@
config.read_string(text)
except AttributeError: # pragma: nocover
# Python 2 has no read_string
config.readfp(io.StringIO(text))
return EntryPoint._from_config(config)
return cls._from_config(config)
@classmethod
def _from_text_for(cls, text, dist):
return (ep._for(dist) for ep in cls._from_text(text))
def _for(self, dist):
self.dist = dist
return self
def __iter__(self):
"""
......@@ -159,6 +165,25 @@
return '<FileHash mode: {} value: {}>'.format(self.mode, self.value)
_T = TypeVar("_T")
class PackageMetadata(Protocol):
def __len__(self) -> int:
... # pragma: no cover
def __contains__(self, item: str) -> bool:
... # pragma: no cover
def __getitem__(self, key: str) -> str:
... # pragma: no cover
def get_all(self, name: str, failobj: _T = ...) -> Union[List[Any], _T]:
"""
Return all values associated with a possibly multi-valued key.
"""
class Distribution:
"""A Python distribution package."""
......@@ -210,8 +235,7 @@
raise ValueError("cannot accept context and kwargs")
context = context or DistributionFinder.Context(**kwargs)
return itertools.chain.from_iterable(
resolver(context)
for resolver in cls._discover_resolvers()
resolver(context) for resolver in cls._discover_resolvers()
)
@staticmethod
......@@ -227,11 +251,10 @@
def _discover_resolvers():
"""Search the meta_path for resolvers."""
declared = (
getattr(finder, 'find_distributions', None)
for finder in sys.meta_path
getattr(finder, 'find_distributions', None) for finder in sys.meta_path
)
return filter(None, declared)
@classmethod
def _local(cls, root='.'):
from pep517 import build, meta
......@@ -232,9 +255,10 @@
)
return filter(None, declared)
@classmethod
def _local(cls, root='.'):
from pep517 import build, meta
system = build.compat_system(root)
builder = functools.partial(
meta.build,
......@@ -244,7 +268,7 @@
return PathDistribution(zipfile.Path(meta.build_as_zip(builder)))
@property
def metadata(self):
def metadata(self) -> PackageMetadata:
"""Return the parsed metadata for this Distribution.
The returned object will have keys that name the various bits of
......@@ -261,9 +285,14 @@
return email.message_from_string(text)
@property
def name(self):
"""Return the 'Name' metadata for the distribution package."""
return self.metadata['Name']
@property
def version(self):
"""Return the 'Version' metadata for the distribution package."""
return self.metadata['Version']
@property
def entry_points(self):
......@@ -264,10 +293,10 @@
def version(self):
"""Return the 'Version' metadata for the distribution package."""
return self.metadata['Version']
@property
def entry_points(self):
return EntryPoint._from_text(self.read_text('entry_points.txt'))
return list(EntryPoint._from_text_for(self.read_text('entry_points.txt'), self))
@property
def files(self):
......@@ -324,8 +353,9 @@
section_pairs = cls._read_sections(source.splitlines())
sections = {
section: list(map(operator.itemgetter('line'), results))
for section, results in
itertools.groupby(section_pairs, operator.itemgetter('section'))
for section, results in itertools.groupby(
section_pairs, operator.itemgetter('section')
)
}
return cls._convert_egg_info_reqs_to_simple_reqs(sections)
......@@ -350,6 +380,7 @@
requirement. This method converts the former to the
latter. See _test_deps_from_requires_text for an example.
"""
def make_condition(name):
return name and 'extra == "{name}"'.format(name=name)
......@@ -438,16 +469,6 @@
names = zip_path.root.namelist()
self.joinpath = zip_path.joinpath
return dict.fromkeys(
child.split(posixpath.sep, 1)[0]
for child in names
)
def is_egg(self, search):
base = self.base
return (
base == search.versionless_egg_name
or base.startswith(search.prefix)
and base.endswith('.egg'))
return dict.fromkeys(child.split(posixpath.sep, 1)[0] for child in names)
def search(self, name):
......@@ -452,16 +473,13 @@
def search(self, name):
for child in self.children():
n_low = child.lower()
if (n_low in name.exact_matches
or n_low.startswith(name.prefix)
and n_low.endswith(name.suffixes)
# legacy case:
or self.is_egg(name) and n_low == 'egg-info'):
yield self.joinpath(child)
return (
self.joinpath(child)
for child in self.children()
if name.matches(child, self.base)
)
class Prepared:
"""
A prepared search for metadata on a possibly-named package.
"""
......@@ -462,10 +480,10 @@
class Prepared:
"""
A prepared search for metadata on a possibly-named package.
"""
normalized = ''
prefix = ''
normalized = None
suffixes = '.dist-info', '.egg-info'
exact_matches = [''][:0]
......@@ -470,8 +488,7 @@
suffixes = '.dist-info', '.egg-info'
exact_matches = [''][:0]
versionless_egg_name = ''
def __init__(self, name):
self.name = name
if name is None:
return
......@@ -473,13 +490,48 @@
def __init__(self, name):
self.name = name
if name is None:
return
self.normalized = name.lower().replace('-', '_')
self.prefix = self.normalized + '-'
self.exact_matches = [
self.normalized + suffix for suffix in self.suffixes]
self.versionless_egg_name = self.normalized + '.egg'
self.normalized = self.normalize(name)
self.exact_matches = [self.normalized + suffix for suffix in self.suffixes]
@staticmethod
def normalize(name):
"""
PEP 503 normalization plus dashes as underscores.
"""
return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_')
@staticmethod
def legacy_normalize(name):
"""
Normalize the package name as found in the convention in
older packaging tools versions and specs.
"""
return name.lower().replace('-', '_')
def matches(self, cand, base):
low = cand.lower()
pre, ext = os.path.splitext(low)
name, sep, rest = pre.partition('-')
return (
low in self.exact_matches
or ext in self.suffixes
and (not self.normalized or name.replace('.', '_') == self.normalized)
# legacy case:
or self.is_egg(base)
and low == 'egg-info'
)
def is_egg(self, base):
normalized = self.legacy_normalize(self.name or '')
prefix = normalized + '-' if normalized else ''
versionless_egg_name = normalized + '.egg' if self.name else ''
return (
base == versionless_egg_name
or base.startswith(prefix)
and base.endswith('.egg')
)
class MetadataPathFinder(DistributionFinder):
......@@ -500,8 +552,7 @@
def _search_paths(cls, name, paths):
"""Find metadata directories in paths heuristically."""
return itertools.chain.from_iterable(
path.search(Prepared(name))
for path in map(FastPath, paths)
path.search(Prepared(name)) for path in map(FastPath, paths)
)
......@@ -515,6 +566,11 @@
self._path = path
def read_text(self, filename):
with suppress(FileNotFoundError, IsADirectoryError, KeyError,
NotADirectoryError, PermissionError):
with suppress(
FileNotFoundError,
IsADirectoryError,
KeyError,
NotADirectoryError,
PermissionError,
):
return self._path.joinpath(filename).read_text(encoding='utf-8')
......@@ -520,4 +576,5 @@
return self._path.joinpath(filename).read_text(encoding='utf-8')
read_text.__doc__ = Distribution.read_text.__doc__
def locate_file(self, path):
......@@ -541,7 +598,7 @@
return Distribution.discover(**kwargs)
def metadata(distribution_name):
def metadata(distribution_name) -> PackageMetadata:
"""Get the metadata for the named package.
:param distribution_name: The name of the distribution package to query.
......@@ -545,7 +602,7 @@
"""Get the metadata for the named package.
:param distribution_name: The name of the distribution package to query.
:return: An email.Message containing the parsed metadata.
:return: A PackageMetadata containing the parsed metadata.
"""
return Distribution.from_name(distribution_name).metadata
......@@ -565,8 +622,7 @@
:return: EntryPoint objects for all installed packages.
"""
eps = itertools.chain.from_iterable(
dist.entry_points for dist in distributions())
eps = itertools.chain.from_iterable(dist.entry_points for dist in distributions())
by_group = operator.attrgetter('group')
ordered = sorted(eps, key=by_group)
grouped = itertools.groupby(ordered, by_group)
......@@ -570,10 +626,7 @@
by_group = operator.attrgetter('group')
ordered = sorted(eps, key=by_group)
grouped = itertools.groupby(ordered, by_group)
return {
group: tuple(eps)
for group, eps in grouped
}
return {group: tuple(eps) for group, eps in grouped}
def files(distribution_name):
......
......@@ -7,6 +7,7 @@
import contextlib
from test.support.os_helper import FS_NONASCII
from typing import Dict, Union
@contextlib.contextmanager
......@@ -71,4 +72,9 @@
self.fixtures.enter_context(self.add_sys_path(self.site_dir))
# Except for python/mypy#731, prefer to define
# FilesDef = Dict[str, Union['FilesDef', str]]
FilesDef = Dict[str, Union[Dict[str, Union[Dict[str, str], str]], str]]
class DistInfoPkg(OnSysPath, SiteDir):
......@@ -74,5 +80,5 @@
class DistInfoPkg(OnSysPath, SiteDir):
files = {
files: FilesDef = {
"distinfo_pkg-1.0.0.dist-info": {
"METADATA": """
Name: distinfo-pkg
......@@ -86,7 +92,7 @@
[entries]
main = mod:main
ns:sub = mod:main
"""
""",
},
"mod.py": """
def main():
......@@ -99,6 +105,42 @@
build_files(DistInfoPkg.files, self.site_dir)
class DistInfoPkgWithDot(OnSysPath, SiteDir):
files: FilesDef = {
"pkg_dot-1.0.0.dist-info": {
"METADATA": """
Name: pkg.dot
Version: 1.0.0
""",
},
}
def setUp(self):
super(DistInfoPkgWithDot, self).setUp()
build_files(DistInfoPkgWithDot.files, self.site_dir)
class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir):
files: FilesDef = {
"pkg.dot-1.0.0.dist-info": {
"METADATA": """
Name: pkg.dot
Version: 1.0.0
""",
},
"pkg.lot.egg-info": {
"METADATA": """
Name: pkg.lot
Version: 1.0.0
""",
},
}
def setUp(self):
super(DistInfoPkgWithDotLegacy, self).setUp()
build_files(DistInfoPkgWithDotLegacy.files, self.site_dir)
class DistInfoPkgOffPath(SiteDir):
def setUp(self):
super(DistInfoPkgOffPath, self).setUp()
......@@ -106,7 +148,7 @@
class EggInfoPkg(OnSysPath, SiteDir):
files = {
files: FilesDef = {
"egginfo_pkg.egg-info": {
"PKG-INFO": """
Name: egginfo-pkg
......@@ -129,7 +171,7 @@
[test]
pytest
""",
"top_level.txt": "mod\n"
"top_level.txt": "mod\n",
},
"mod.py": """
def main():
......@@ -143,7 +185,7 @@
class EggInfoFile(OnSysPath, SiteDir):
files = {
files: FilesDef = {
"egginfo_file.egg-info": """
Metadata-Version: 1.0
Name: egginfo_file
......@@ -164,7 +206,7 @@
class LocalPackage:
files = {
files: FilesDef = {
"setup.py": """
import setuptools
setuptools.setup(name="local-pkg", version="2.0.1")
......@@ -214,8 +256,7 @@
class FileBuilder:
def unicode_filename(self):
return FS_NONASCII or \
self.skip("File system does not support non-ascii.")
return FS_NONASCII or self.skip("File system does not support non-ascii.")
def DALS(str):
......
# coding: utf-8
import re
import json
import pickle
......@@ -14,9 +12,13 @@
from . import fixtures
from importlib.metadata import (
Distribution, EntryPoint,
PackageNotFoundError, distributions,
entry_points, metadata, version,
Distribution,
EntryPoint,
PackageNotFoundError,
distributions,
entry_points,
metadata,
version,
)
......@@ -74,8 +76,7 @@
assert ep.load() is importlib.metadata
class NameNormalizationTests(
fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
@staticmethod
def pkg_with_dashes(site_dir):
"""
......@@ -144,7 +145,9 @@
metadata_dir.mkdir()
metadata = metadata_dir / 'METADATA'
with metadata.open('w', encoding='utf-8') as fp:
fp.write(textwrap.dedent("""
fp.write(
textwrap.dedent(
"""
Name: portend
pôrˈtend
......@@ -148,7 +151,9 @@
Name: portend
pôrˈtend
""").lstrip())
"""
).lstrip()
)
return 'portend'
def test_metadata_loads(self):
......@@ -162,9 +167,6 @@
assert meta.get_payload() == 'pôrˈtend\n'
class DiscoveryTests(fixtures.EggInfoPkg,
fixtures.DistInfoPkg,
unittest.TestCase):
class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase):
def test_package_discovery(self):
dists = list(distributions())
......@@ -169,17 +171,8 @@
def test_package_discovery(self):
dists = list(distributions())
assert all(
isinstance(dist, Distribution)
for dist in dists
)
assert any(
dist.metadata['Name'] == 'egginfo-pkg'
for dist in dists
)
assert any(
dist.metadata['Name'] == 'distinfo-pkg'
for dist in dists
)
assert all(isinstance(dist, Distribution) for dist in dists)
assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists)
assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists)
def test_invalid_usage(self):
with self.assertRaises(ValueError):
......@@ -265,5 +258,16 @@
def test_attr(self):
assert self.ep.attr is None
def test_sortable(self):
"""
EntryPoint objects are sortable, but result is undefined.
"""
sorted(
[
EntryPoint('b', 'val', 'group'),
EntryPoint('a', 'val', 'group'),
]
)
class FileSystem(
......@@ -268,7 +272,7 @@
class FileSystem(
fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder,
unittest.TestCase):
fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase
):
def test_unicode_dir_on_sys_path(self):
"""
Ensure a Unicode subdirectory of a directory on sys.path
......
......@@ -2,7 +2,5 @@
import textwrap
import unittest
from collections.abc import Iterator
from . import fixtures
from importlib.metadata import (
......@@ -7,10 +5,16 @@
from . import fixtures
from importlib.metadata import (
Distribution, PackageNotFoundError, distribution,
entry_points, files, metadata, requires, version,
Distribution,
PackageNotFoundError,
distribution,
entry_points,
files,
metadata,
requires,
version,
)
class APITests(
fixtures.EggInfoPkg,
fixtures.DistInfoPkg,
......@@ -11,7 +15,8 @@
)
class APITests(
fixtures.EggInfoPkg,
fixtures.DistInfoPkg,
fixtures.DistInfoPkgWithDot,
fixtures.EggInfoFile,
......@@ -17,5 +22,6 @@
fixtures.EggInfoFile,
unittest.TestCase):
unittest.TestCase,
):
version_pattern = r'\d+\.\d+(\.\d)?'
......@@ -33,5 +39,18 @@
with self.assertRaises(PackageNotFoundError):
distribution('does-not-exist')
def test_name_normalization(self):
names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot'
for name in names:
with self.subTest(name):
assert distribution(name).metadata['Name'] == 'pkg.dot'
def test_prefix_not_matched(self):
prefixes = 'p', 'pkg', 'pkg.'
for prefix in prefixes:
with self.subTest(prefix):
with self.assertRaises(PackageNotFoundError):
distribution(prefix)
def test_for_top_level(self):
self.assertEqual(
......@@ -36,7 +55,7 @@
def test_for_top_level(self):
self.assertEqual(
distribution('egginfo-pkg').read_text('top_level.txt').strip(),
'mod')
distribution('egginfo-pkg').read_text('top_level.txt').strip(), 'mod'
)
def test_read_text(self):
top_level = [
......@@ -40,8 +59,7 @@
def test_read_text(self):
top_level = [
path for path in files('egginfo-pkg')
if path.name == 'top_level.txt'
path for path in files('egginfo-pkg') if path.name == 'top_level.txt'
][0]
self.assertEqual(top_level.read_text(), 'mod\n')
......@@ -51,6 +69,13 @@
self.assertEqual(ep.value, 'mod:main')
self.assertEqual(ep.extras, [])
def test_entry_points_distribution(self):
entries = dict(entry_points()['entries'])
for entry in ("main", "ns:sub"):
ep = entries[entry]
self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg'))
self.assertEqual(ep.dist.version, "1.0.0")
def test_metadata_for_this_package(self):
md = metadata('egginfo-pkg')
assert md['author'] == 'Steven Ma'
......@@ -75,13 +100,8 @@
def test_file_hash_repr(self):
assertRegex = self.assertRegex
util = [
p for p in files('distinfo-pkg')
if p.name == 'mod.py'
][0]
assertRegex(
repr(util.hash),
'<FileHash mode: sha256 value: .*>')
util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0]
assertRegex(repr(util.hash), '<FileHash mode: sha256 value: .*>')
def test_files_dist_info(self):
self._test_files(files('distinfo-pkg'))
......@@ -99,10 +119,7 @@
def test_requires_egg_info(self):
deps = requires('egginfo-pkg')
assert len(deps) == 2
assert any(
dep == 'wheel >= 1.0; python_version >= "2.7"'
for dep in deps
)
assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps)
def test_requires_dist_info(self):
deps = requires('distinfo-pkg')
......@@ -112,7 +129,8 @@
assert "pytest; extra == 'test'" in deps
def test_more_complex_deps_requires_text(self):
requires = textwrap.dedent("""
requires = textwrap.dedent(
"""
dep1
dep2
......@@ -124,7 +142,8 @@
[extra2:python_version < "3"]
dep5
""")
"""
)
deps = sorted(Distribution._deps_from_requires_text(requires))
expected = [
'dep1',
......@@ -140,6 +159,20 @@
assert deps == expected
class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase):
def test_name_normalization(self):
names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot'
for name in names:
with self.subTest(name):
assert distribution(name).metadata['Name'] == 'pkg.dot'
def test_name_normalization_versionless_egg_info(self):
names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot'
for name in names:
with self.subTest(name):
assert distribution(name).metadata['Name'] == 'pkg.lot'
class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase):
def test_find_distributions_specified_path(self):
dists = Distribution.discover(path=[str(self.site_dir)])
......@@ -143,9 +176,6 @@
class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase):
def test_find_distributions_specified_path(self):
dists = Distribution.discover(path=[str(self.site_dir)])
assert any(
dist.metadata['Name'] == 'distinfo-pkg'
for dist in dists
)
assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists)
def test_distribution_at_pathlib(self):
......@@ -150,7 +180,6 @@
def test_distribution_at_pathlib(self):
"""Demonstrate how to load metadata direct from a directory.
"""
"""Demonstrate how to load metadata direct from a directory."""
dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info'
dist = Distribution.at(dist_info_path)
assert dist.version == '1.0.0'
......
......@@ -3,8 +3,12 @@
from contextlib import ExitStack
from importlib.metadata import (
distribution, entry_points, files, PackageNotFoundError,
version, distributions,
PackageNotFoundError,
distribution,
distributions,
entry_points,
files,
version,
)
from importlib import resources
......
In ``importlib.metadata``: - ``EntryPoint`` objects now expose a ``.dist``
object referencing the ``Distribution`` when constructed from a
``Distribution``. - Add support for package discovery under package
normalization rules. - The object returned by ``metadata()`` now has a
formally-defined protocol called ``PackageMetadata`` with declared support
for the ``.get_all()`` method. - Synced with importlib_metadata 3.3.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment