codekingpro/portable-devtools

1# -*- coding: utf-8 -*-2#3# Copyright (C) 2013-2023 Vinay Sajip.4# Licensed to the Python Software Foundation under a contributor agreement.5# See LICENSE.txt and CONTRIBUTORS.txt.6#7import hashlib8import logging9import os10import shutil11import subprocess12import tempfile13try:14    from threading import Thread15except ImportError:  # pragma: no cover16    from dummy_threading import Thread17 18from . import DistlibException19from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr,20                     urlparse, build_opener, string_types)21from .util import zip_dir, ServerProxy22 23logger = logging.getLogger(__name__)24 25DEFAULT_INDEX = 'https://pypi.org/pypi'26DEFAULT_REALM = 'pypi'27 28 29class PackageIndex(object):30    """31    This class represents a package index compatible with PyPI, the Python32    Package Index.33    """34 35    boundary = b'----------ThIs_Is_tHe_distlib_index_bouNdaRY_$'36 37    def __init__(self, url=None):38        """39        Initialise an instance.40 41        :param url: The URL of the index. If not specified, the URL for PyPI is42                    used.43        """44        self.url = url or DEFAULT_INDEX45        self.read_configuration()46        scheme, netloc, path, params, query, frag = urlparse(self.url)47        if params or query or frag or scheme not in ('http', 'https'):48            raise DistlibException('invalid repository: %s' % self.url)49        self.password_handler = None50        self.ssl_verifier = None51        self.gpg = None52        self.gpg_home = None53        with open(os.devnull, 'w') as sink:54            # Use gpg by default rather than gpg2, as gpg2 insists on55            # prompting for passwords56            for s in ('gpg', 'gpg2'):57                try:58                    rc = subprocess.check_call([s, '--version'], stdout=sink,59                                               stderr=sink)60                    if rc == 0:61                        self.gpg = s62                        break63                except OSError:64                    pass65 66    def _get_pypirc_command(self):67        """68        Get the distutils command for interacting with PyPI configurations.69        :return: the command.70        """71        from .util import _get_pypirc_command as cmd72        return cmd()73 74    def read_configuration(self):75        """76        Read the PyPI access configuration as supported by distutils. This populates77        ``username``, ``password``, ``realm`` and ``url`` attributes from the78        configuration.79        """80        from .util import _load_pypirc81        cfg = _load_pypirc(self)82        self.username = cfg.get('username')83        self.password = cfg.get('password')84        self.realm = cfg.get('realm', 'pypi')85        self.url = cfg.get('repository', self.url)86 87    def save_configuration(self):88        """89        Save the PyPI access configuration. You must have set ``username`` and90        ``password`` attributes before calling this method.91        """92        self.check_credentials()93        from .util import _store_pypirc94        _store_pypirc(self)95 96    def check_credentials(self):97        """98        Check that ``username`` and ``password`` have been set, and raise an99        exception if not.100        """101        if self.username is None or self.password is None:102            raise DistlibException('username and password must be set')103        pm = HTTPPasswordMgr()104        _, netloc, _, _, _, _ = urlparse(self.url)105        pm.add_password(self.realm, netloc, self.username, self.password)106        self.password_handler = HTTPBasicAuthHandler(pm)107 108    def register(self, metadata):  # pragma: no cover109        """110        Register a distribution on PyPI, using the provided metadata.111 112        :param metadata: A :class:`Metadata` instance defining at least a name113                         and version number for the distribution to be114                         registered.115        :return: The HTTP response received from PyPI upon submission of the116                request.117        """118        self.check_credentials()119        metadata.validate()120        d = metadata.todict()121        d[':action'] = 'verify'122        request = self.encode_request(d.items(), [])123        self.send_request(request)124        d[':action'] = 'submit'125        request = self.encode_request(d.items(), [])126        return self.send_request(request)127 128    def _reader(self, name, stream, outbuf):129        """130        Thread runner for reading lines of from a subprocess into a buffer.131 132        :param name: The logical name of the stream (used for logging only).133        :param stream: The stream to read from. This will typically a pipe134                       connected to the output stream of a subprocess.135        :param outbuf: The list to append the read lines to.136        """137        while True:138            s = stream.readline()139            if not s:140                break141            s = s.decode('utf-8').rstrip()142            outbuf.append(s)143            logger.debug('%s: %s' % (name, s))144        stream.close()145 146    def get_sign_command(self, filename, signer, sign_password, keystore=None):  # pragma: no cover147        """148        Return a suitable command for signing a file.149 150        :param filename: The pathname to the file to be signed.151        :param signer: The identifier of the signer of the file.152        :param sign_password: The passphrase for the signer's153                              private key used for signing.154        :param keystore: The path to a directory which contains the keys155                         used in verification. If not specified, the156                         instance's ``gpg_home`` attribute is used instead.157        :return: The signing command as a list suitable to be158                 passed to :class:`subprocess.Popen`.159        """160        cmd = [self.gpg, '--status-fd', '2', '--no-tty']161        if keystore is None:162            keystore = self.gpg_home163        if keystore:164            cmd.extend(['--homedir', keystore])165        if sign_password is not None:166            cmd.extend(['--batch', '--passphrase-fd', '0'])167        td = tempfile.mkdtemp()168        sf = os.path.join(td, os.path.basename(filename) + '.asc')169        cmd.extend(['--detach-sign', '--armor', '--local-user',170                    signer, '--output', sf, filename])171        logger.debug('invoking: %s', ' '.join(cmd))172        return cmd, sf173 174    def run_command(self, cmd, input_data=None):175        """176        Run a command in a child process , passing it any input data specified.177 178        :param cmd: The command to run.179        :param input_data: If specified, this must be a byte string containing180                           data to be sent to the child process.181        :return: A tuple consisting of the subprocess' exit code, a list of182                 lines read from the subprocess' ``stdout``, and a list of183                 lines read from the subprocess' ``stderr``.184        """185        kwargs = {186            'stdout': subprocess.PIPE,187            'stderr': subprocess.PIPE,188        }189        if input_data is not None:190            kwargs['stdin'] = subprocess.PIPE191        stdout = []192        stderr = []193        p = subprocess.Popen(cmd, **kwargs)194        # We don't use communicate() here because we may need to195        # get clever with interacting with the command196        t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout))197        t1.start()198        t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr))199        t2.start()200        if input_data is not None:201            p.stdin.write(input_data)202            p.stdin.close()203 204        p.wait()205        t1.join()206        t2.join()207        return p.returncode, stdout, stderr208 209    def sign_file(self, filename, signer, sign_password, keystore=None):  # pragma: no cover210        """211        Sign a file.212 213        :param filename: The pathname to the file to be signed.214        :param signer: The identifier of the signer of the file.215        :param sign_password: The passphrase for the signer's216                              private key used for signing.217        :param keystore: The path to a directory which contains the keys218                         used in signing. If not specified, the instance's219                         ``gpg_home`` attribute is used instead.220        :return: The absolute pathname of the file where the signature is221                 stored.222        """223        cmd, sig_file = self.get_sign_command(filename, signer, sign_password,224                                              keystore)225        rc, stdout, stderr = self.run_command(cmd,226                                              sign_password.encode('utf-8'))227        if rc != 0:228            raise DistlibException('sign command failed with error '229                                   'code %s' % rc)230        return sig_file231 232    def upload_file(self, metadata, filename, signer=None, sign_password=None,233                    filetype='sdist', pyversion='source', keystore=None):234        """235        Upload a release file to the index.236 237        :param metadata: A :class:`Metadata` instance defining at least a name238                         and version number for the file to be uploaded.239        :param filename: The pathname of the file to be uploaded.240        :param signer: The identifier of the signer of the file.241        :param sign_password: The passphrase for the signer's242                              private key used for signing.243        :param filetype: The type of the file being uploaded. This is the244                        distutils command which produced that file, e.g.245                        ``sdist`` or ``bdist_wheel``.246        :param pyversion: The version of Python which the release relates247                          to. For code compatible with any Python, this would248                          be ``source``, otherwise it would be e.g. ``3.2``.249        :param keystore: The path to a directory which contains the keys250                         used in signing. If not specified, the instance's251                         ``gpg_home`` attribute is used instead.252        :return: The HTTP response received from PyPI upon submission of the253                request.254        """255        self.check_credentials()256        if not os.path.exists(filename):257            raise DistlibException('not found: %s' % filename)258        metadata.validate()259        d = metadata.todict()260        sig_file = None261        if signer:262            if not self.gpg:263                logger.warning('no signing program available - not signed')264            else:265                sig_file = self.sign_file(filename, signer, sign_password,266                                          keystore)267        with open(filename, 'rb') as f:268            file_data = f.read()269        md5_digest = hashlib.md5(file_data).hexdigest()270        sha256_digest = hashlib.sha256(file_data).hexdigest()271        d.update({272            ':action': 'file_upload',273            'protocol_version': '1',274            'filetype': filetype,275            'pyversion': pyversion,276            'md5_digest': md5_digest,277            'sha256_digest': sha256_digest,278        })279        files = [('content', os.path.basename(filename), file_data)]280        if sig_file:281            with open(sig_file, 'rb') as f:282                sig_data = f.read()283            files.append(('gpg_signature', os.path.basename(sig_file),284                         sig_data))285            shutil.rmtree(os.path.dirname(sig_file))286        request = self.encode_request(d.items(), files)287        return self.send_request(request)288 289    def upload_documentation(self, metadata, doc_dir):  # pragma: no cover290        """291        Upload documentation to the index.292 293        :param metadata: A :class:`Metadata` instance defining at least a name294                         and version number for the documentation to be295                         uploaded.296        :param doc_dir: The pathname of the directory which contains the297                        documentation. This should be the directory that298                        contains the ``index.html`` for the documentation.299        :return: The HTTP response received from PyPI upon submission of the300                request.301        """302        self.check_credentials()303        if not os.path.isdir(doc_dir):304            raise DistlibException('not a directory: %r' % doc_dir)305        fn = os.path.join(doc_dir, 'index.html')306        if not os.path.exists(fn):307            raise DistlibException('not found: %r' % fn)308        metadata.validate()309        name, version = metadata.name, metadata.version310        zip_data = zip_dir(doc_dir).getvalue()311        fields = [(':action', 'doc_upload'),312                  ('name', name), ('version', version)]313        files = [('content', name, zip_data)]314        request = self.encode_request(fields, files)315        return self.send_request(request)316 317    def get_verify_command(self, signature_filename, data_filename,318                           keystore=None):319        """320        Return a suitable command for verifying a file.321 322        :param signature_filename: The pathname to the file containing the323                                   signature.324        :param data_filename: The pathname to the file containing the325                              signed data.326        :param keystore: The path to a directory which contains the keys327                         used in verification. If not specified, the328                         instance's ``gpg_home`` attribute is used instead.329        :return: The verifying command as a list suitable to be330                 passed to :class:`subprocess.Popen`.331        """332        cmd = [self.gpg, '--status-fd', '2', '--no-tty']333        if keystore is None:334            keystore = self.gpg_home335        if keystore:336            cmd.extend(['--homedir', keystore])337        cmd.extend(['--verify', signature_filename, data_filename])338        logger.debug('invoking: %s', ' '.join(cmd))339        return cmd340 341    def verify_signature(self, signature_filename, data_filename,342                         keystore=None):343        """344        Verify a signature for a file.345 346        :param signature_filename: The pathname to the file containing the347                                   signature.348        :param data_filename: The pathname to the file containing the349                              signed data.350        :param keystore: The path to a directory which contains the keys351                         used in verification. If not specified, the352                         instance's ``gpg_home`` attribute is used instead.353        :return: True if the signature was verified, else False.354        """355        if not self.gpg:356            raise DistlibException('verification unavailable because gpg '357                                   'unavailable')358        cmd = self.get_verify_command(signature_filename, data_filename,359                                      keystore)360        rc, stdout, stderr = self.run_command(cmd)361        if rc not in (0, 1):362            raise DistlibException('verify command failed with error code %s' % rc)363        return rc == 0364 365    def download_file(self, url, destfile, digest=None, reporthook=None):366        """367        This is a convenience method for downloading a file from an URL.368        Normally, this will be a file from the index, though currently369        no check is made for this (i.e. a file can be downloaded from370        anywhere).371 372        The method is just like the :func:`urlretrieve` function in the373        standard library, except that it allows digest computation to be374        done during download and checking that the downloaded data375        matched any expected value.376 377        :param url: The URL of the file to be downloaded (assumed to be378                    available via an HTTP GET request).379        :param destfile: The pathname where the downloaded file is to be380                         saved.381        :param digest: If specified, this must be a (hasher, value)382                       tuple, where hasher is the algorithm used (e.g.383                       ``'md5'``) and ``value`` is the expected value.384        :param reporthook: The same as for :func:`urlretrieve` in the385                           standard library.386        """387        if digest is None:388            digester = None389            logger.debug('No digest specified')390        else:391            if isinstance(digest, (list, tuple)):392                hasher, digest = digest393            else:394                hasher = 'md5'395            digester = getattr(hashlib, hasher)()396            logger.debug('Digest specified: %s' % digest)397        # The following code is equivalent to urlretrieve.398        # We need to do it this way so that we can compute the399        # digest of the file as we go.400        with open(destfile, 'wb') as dfp:401            # addinfourl is not a context manager on 2.x402            # so we have to use try/finally403            sfp = self.send_request(Request(url))404            try:405                headers = sfp.info()406                blocksize = 8192407                size = -1408                read = 0409                blocknum = 0410                if "content-length" in headers:411                    size = int(headers["Content-Length"])412                if reporthook:413                    reporthook(blocknum, blocksize, size)414                while True:415                    block = sfp.read(blocksize)416                    if not block:417                        break418                    read += len(block)419                    dfp.write(block)420                    if digester:421                        digester.update(block)422                    blocknum += 1423                    if reporthook:424                        reporthook(blocknum, blocksize, size)425            finally:426                sfp.close()427 428        # check that we got the whole file, if we can429        if size >= 0 and read < size:430            raise DistlibException(431                'retrieval incomplete: got only %d out of %d bytes'432                % (read, size))433        # if we have a digest, it must match.434        if digester:435            actual = digester.hexdigest()436            if digest != actual:437                raise DistlibException('%s digest mismatch for %s: expected '438                                       '%s, got %s' % (hasher, destfile,439                                                       digest, actual))440            logger.debug('Digest verified: %s', digest)441 442    def send_request(self, req):443        """444        Send a standard library :class:`Request` to PyPI and return its445        response.446 447        :param req: The request to send.448        :return: The HTTP response from PyPI (a standard library HTTPResponse).449        """450        handlers = []451        if self.password_handler:452            handlers.append(self.password_handler)453        if self.ssl_verifier:454            handlers.append(self.ssl_verifier)455        opener = build_opener(*handlers)456        return opener.open(req)457 458    def encode_request(self, fields, files):459        """460        Encode fields and files for posting to an HTTP server.461 462        :param fields: The fields to send as a list of (fieldname, value)463                       tuples.464        :param files: The files to send as a list of (fieldname, filename,465                      file_bytes) tuple.466        """467        # Adapted from packaging, which in turn was adapted from468        # http://code.activestate.com/recipes/146306469 470        parts = []471        boundary = self.boundary472        for k, values in fields:473            if not isinstance(values, (list, tuple)):474                values = [values]475 476            for v in values:477                parts.extend((478                    b'--' + boundary,479                    ('Content-Disposition: form-data; name="%s"' %480                     k).encode('utf-8'),481                    b'',482                    v.encode('utf-8')))483        for key, filename, value in files:484            parts.extend((485                b'--' + boundary,486                ('Content-Disposition: form-data; name="%s"; filename="%s"' %487                 (key, filename)).encode('utf-8'),488                b'',489                value))490 491        parts.extend((b'--' + boundary + b'--', b''))492 493        body = b'\r\n'.join(parts)494        ct = b'multipart/form-data; boundary=' + boundary495        headers = {496            'Content-type': ct,497            'Content-length': str(len(body))498        }499        return Request(self.url, body, headers)500 501    def search(self, terms, operator=None):  # pragma: no cover502        if isinstance(terms, string_types):503            terms = {'name': terms}504        rpc_proxy = ServerProxy(self.url, timeout=3.0)505        try:506            return rpc_proxy.search(terms, operator or 'and')507        finally:508            rpc_proxy('close')()509