codekingpro/portable-devtools
114k
1# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License"). You4# may not use this file except in compliance with the License. A copy of5# the License is located at6#7# http://aws.amazon.com/apache2.0/8#9# or in the "license" file accompanying this file. This file is10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF11# ANY KIND, either express or implied. See the License for the specific12# language governing permissions and limitations under the License.13import base6414import binascii15import datetime16import email.message17import functools18import hashlib19import io20import logging21import os22import random23import re24import socket25import time26import warnings27import weakref28from datetime import datetime as _DatetimeClass29from ipaddress import ip_address30from pathlib import Path31from urllib.request import getproxies, proxy_bypass32 33import dateutil.parser34from dateutil.tz import tzutc35from urllib3.exceptions import LocationParseError36 37import botocore38import botocore.awsrequest39import botocore.httpsession40 41# IP Regexes retained for backwards compatibility42from botocore.compat import HEX_PAT # noqa: F40143from botocore.compat import IPV4_PAT # noqa: F40144from botocore.compat import IPV6_ADDRZ_PAT # noqa: F40145from botocore.compat import IPV6_PAT # noqa: F40146from botocore.compat import LS32_PAT # noqa: F40147from botocore.compat import UNRESERVED_PAT # noqa: F40148from botocore.compat import ZONE_ID_PAT # noqa: F40149from botocore.compat import (50 HAS_CRT,51 IPV4_RE,52 IPV6_ADDRZ_RE,53 MD5_AVAILABLE,54 UNSAFE_URL_CHARS,55 OrderedDict,56 get_md5,57 get_tzinfo_options,58 json,59 quote,60 urlparse,61 urlsplit,62 urlunsplit,63 zip_longest,64)65from botocore.exceptions import (66 ClientError,67 ConfigNotFound,68 ConnectionClosedError,69 ConnectTimeoutError,70 EndpointConnectionError,71 HTTPClientError,72 InvalidDNSNameError,73 InvalidEndpointConfigurationError,74 InvalidExpressionError,75 InvalidHostLabelError,76 InvalidIMDSEndpointError,77 InvalidIMDSEndpointModeError,78 InvalidRegionError,79 MetadataRetrievalError,80 MissingDependencyException,81 ReadTimeoutError,82 SSOTokenLoadError,83 UnsupportedOutpostResourceError,84 UnsupportedS3AccesspointConfigurationError,85 UnsupportedS3ArnError,86 UnsupportedS3ConfigurationError,87 UnsupportedS3ControlArnError,88 UnsupportedS3ControlConfigurationError,89)90 91logger = logging.getLogger(__name__)92DEFAULT_METADATA_SERVICE_TIMEOUT = 193METADATA_BASE_URL = 'http://169.254.169.254/'94METADATA_BASE_URL_IPv6 = 'http://[fd00:ec2::254]/'95METADATA_ENDPOINT_MODES = ('ipv4', 'ipv6')96 97# These are chars that do not need to be urlencoded.98# Based on rfc2986, section 2.399SAFE_CHARS = '-._~'100LABEL_RE = re.compile(r'[a-z0-9][a-z0-9\-]*[a-z0-9]')101RETRYABLE_HTTP_ERRORS = (102 ReadTimeoutError,103 EndpointConnectionError,104 ConnectionClosedError,105 ConnectTimeoutError,106)107S3_ACCELERATE_WHITELIST = ['dualstack']108# In switching events from using service name / endpoint prefix to service109# id, we have to preserve compatibility. This maps the instances where either110# is different than the transformed service id.111EVENT_ALIASES = {112 "a4b": "alexa-for-business",113 "alexaforbusiness": "alexa-for-business",114 "api.mediatailor": "mediatailor",115 "api.pricing": "pricing",116 "api.sagemaker": "sagemaker",117 "apigateway": "api-gateway",118 "application-autoscaling": "application-auto-scaling",119 "appstream2": "appstream",120 "autoscaling": "auto-scaling",121 "autoscaling-plans": "auto-scaling-plans",122 "ce": "cost-explorer",123 "cloudhsmv2": "cloudhsm-v2",124 "cloudsearchdomain": "cloudsearch-domain",125 "cognito-idp": "cognito-identity-provider",126 "config": "config-service",127 "cur": "cost-and-usage-report-service",128 "data.iot": "iot-data-plane",129 "data.jobs.iot": "iot-jobs-data-plane",130 "data.mediastore": "mediastore-data",131 "datapipeline": "data-pipeline",132 "devicefarm": "device-farm",133 "devices.iot1click": "iot-1click-devices-service",134 "directconnect": "direct-connect",135 "discovery": "application-discovery-service",136 "dms": "database-migration-service",137 "ds": "directory-service",138 "dynamodbstreams": "dynamodb-streams",139 "elasticbeanstalk": "elastic-beanstalk",140 "elasticfilesystem": "efs",141 "elasticloadbalancing": "elastic-load-balancing",142 "elasticmapreduce": "emr",143 "elastictranscoder": "elastic-transcoder",144 "elb": "elastic-load-balancing",145 "elbv2": "elastic-load-balancing-v2",146 "email": "ses",147 "entitlement.marketplace": "marketplace-entitlement-service",148 "es": "elasticsearch-service",149 "events": "eventbridge",150 "cloudwatch-events": "eventbridge",151 "iot-data": "iot-data-plane",152 "iot-jobs-data": "iot-jobs-data-plane",153 "iot1click-devices": "iot-1click-devices-service",154 "iot1click-projects": "iot-1click-projects",155 "kinesisanalytics": "kinesis-analytics",156 "kinesisvideo": "kinesis-video",157 "lex-models": "lex-model-building-service",158 "lex-runtime": "lex-runtime-service",159 "logs": "cloudwatch-logs",160 "machinelearning": "machine-learning",161 "marketplace-entitlement": "marketplace-entitlement-service",162 "marketplacecommerceanalytics": "marketplace-commerce-analytics",163 "metering.marketplace": "marketplace-metering",164 "meteringmarketplace": "marketplace-metering",165 "mgh": "migration-hub",166 "models.lex": "lex-model-building-service",167 "monitoring": "cloudwatch",168 "mturk-requester": "mturk",169 "opsworks-cm": "opsworkscm",170 "projects.iot1click": "iot-1click-projects",171 "resourcegroupstaggingapi": "resource-groups-tagging-api",172 "route53": "route-53",173 "route53domains": "route-53-domains",174 "runtime.lex": "lex-runtime-service",175 "runtime.sagemaker": "sagemaker-runtime",176 "sdb": "simpledb",177 "secretsmanager": "secrets-manager",178 "serverlessrepo": "serverlessapplicationrepository",179 "servicecatalog": "service-catalog",180 "states": "sfn",181 "stepfunctions": "sfn",182 "storagegateway": "storage-gateway",183 "streams.dynamodb": "dynamodb-streams",184 "tagging": "resource-groups-tagging-api",185}186 187 188# This pattern can be used to detect if a header is a flexible checksum header189CHECKSUM_HEADER_PATTERN = re.compile(190 r'^X-Amz-Checksum-([a-z0-9]*)$',191 flags=re.IGNORECASE,192)193 194 195def ensure_boolean(val):196 """Ensures a boolean value if a string or boolean is provided197 198 For strings, the value for True/False is case insensitive199 """200 if isinstance(val, bool):201 return val202 elif isinstance(val, str):203 return val.lower() == 'true'204 else:205 return False206 207 208def resolve_imds_endpoint_mode(session):209 """Resolving IMDS endpoint mode to either IPv6 or IPv4.210 211 ec2_metadata_service_endpoint_mode takes precedence over imds_use_ipv6.212 """213 endpoint_mode = session.get_config_variable(214 'ec2_metadata_service_endpoint_mode'215 )216 if endpoint_mode is not None:217 lendpoint_mode = endpoint_mode.lower()218 if lendpoint_mode not in METADATA_ENDPOINT_MODES:219 error_msg_kwargs = {220 'mode': endpoint_mode,221 'valid_modes': METADATA_ENDPOINT_MODES,222 }223 raise InvalidIMDSEndpointModeError(**error_msg_kwargs)224 return lendpoint_mode225 elif session.get_config_variable('imds_use_ipv6'):226 return 'ipv6'227 return 'ipv4'228 229 230def is_json_value_header(shape):231 """Determines if the provided shape is the special header type jsonvalue.232 233 :type shape: botocore.shape234 :param shape: Shape to be inspected for the jsonvalue trait.235 236 :return: True if this type is a jsonvalue, False otherwise237 :rtype: Bool238 """239 return (240 hasattr(shape, 'serialization')241 and shape.serialization.get('jsonvalue', False)242 and shape.serialization.get('location') == 'header'243 and shape.type_name == 'string'244 )245 246 247def has_header(header_name, headers):248 """Case-insensitive check for header key."""249 if header_name is None:250 return False251 elif isinstance(headers, botocore.awsrequest.HeadersDict):252 return header_name in headers253 else:254 return header_name.lower() in [key.lower() for key in headers.keys()]255 256 257def get_service_module_name(service_model):258 """Returns the module name for a service259 260 This is the value used in both the documentation and client class name261 """262 name = service_model.metadata.get(263 'serviceAbbreviation',264 service_model.metadata.get(265 'serviceFullName', service_model.service_name266 ),267 )268 name = name.replace('Amazon', '')269 name = name.replace('AWS', '')270 name = re.sub(r'\W+', '', name)271 return name272 273 274def normalize_url_path(path):275 if not path:276 return '/'277 return remove_dot_segments(path)278 279 280def normalize_boolean(val):281 """Returns None if val is None, otherwise ensure value282 converted to boolean"""283 if val is None:284 return val285 else:286 return ensure_boolean(val)287 288 289def remove_dot_segments(url):290 # RFC 3986, section 5.2.4 "Remove Dot Segments"291 # Also, AWS services require consecutive slashes to be removed,292 # so that's done here as well293 if not url:294 return ''295 input_url = url.split('/')296 output_list = []297 for x in input_url:298 if x and x != '.':299 if x == '..':300 if output_list:301 output_list.pop()302 else:303 output_list.append(x)304 305 if url[0] == '/':306 first = '/'307 else:308 first = ''309 if url[-1] == '/' and output_list:310 last = '/'311 else:312 last = ''313 return first + '/'.join(output_list) + last314 315 316def validate_jmespath_for_set(expression):317 # Validates a limited jmespath expression to determine if we can set a318 # value based on it. Only works with dotted paths.319 if not expression or expression == '.':320 raise InvalidExpressionError(expression=expression)321 322 for invalid in ['[', ']', '*']:323 if invalid in expression:324 raise InvalidExpressionError(expression=expression)325 326 327def set_value_from_jmespath(source, expression, value, is_first=True):328 # This takes a (limited) jmespath-like expression & can set a value based329 # on it.330 # Limitations:331 # * Only handles dotted lookups332 # * No offsets/wildcards/slices/etc.333 if is_first:334 validate_jmespath_for_set(expression)335 336 bits = expression.split('.', 1)337 current_key, remainder = bits[0], bits[1] if len(bits) > 1 else ''338 339 if not current_key:340 raise InvalidExpressionError(expression=expression)341 342 if remainder:343 if current_key not in source:344 # We've got something in the expression that's not present in the345 # source (new key). If there's any more bits, we'll set the key346 # with an empty dictionary.347 source[current_key] = {}348 349 return set_value_from_jmespath(350 source[current_key], remainder, value, is_first=False351 )352 353 # If we're down to a single key, set it.354 source[current_key] = value355 356 357def is_global_accesspoint(context):358 """Determine if request is intended for an MRAP accesspoint."""359 s3_accesspoint = context.get('s3_accesspoint', {})360 is_global = s3_accesspoint.get('region') == ''361 return is_global362 363 364class _RetriesExceededError(Exception):365 """Internal exception used when the number of retries are exceeded."""366 367 pass368 369 370class BadIMDSRequestError(Exception):371 def __init__(self, request):372 self.request = request373 374 375class IMDSFetcher:376 _RETRIES_EXCEEDED_ERROR_CLS = _RetriesExceededError377 _TOKEN_PATH = 'latest/api/token'378 _TOKEN_TTL = '21600'379 380 def __init__(381 self,382 timeout=DEFAULT_METADATA_SERVICE_TIMEOUT,383 num_attempts=1,384 base_url=METADATA_BASE_URL,385 env=None,386 user_agent=None,387 config=None,388 ):389 self._timeout = timeout390 self._num_attempts = num_attempts391 if config is None:392 config = {}393 self._base_url = self._select_base_url(base_url, config)394 self._config = config395 396 if env is None:397 env = os.environ.copy()398 self._disabled = (399 env.get('AWS_EC2_METADATA_DISABLED', 'false').lower() == 'true'400 )401 self._imds_v1_disabled = config.get('ec2_metadata_v1_disabled')402 self._user_agent = user_agent403 self._session = botocore.httpsession.URLLib3Session(404 timeout=self._timeout,405 proxies=get_environ_proxies(self._base_url),406 )407 408 def get_base_url(self):409 return self._base_url410 411 def _select_base_url(self, base_url, config):412 if config is None:413 config = {}414 415 requires_ipv6 = (416 config.get('ec2_metadata_service_endpoint_mode') == 'ipv6'417 )418 custom_metadata_endpoint = config.get('ec2_metadata_service_endpoint')419 420 if requires_ipv6 and custom_metadata_endpoint:421 logger.warning(422 "Custom endpoint and IMDS_USE_IPV6 are both set. Using custom endpoint."423 )424 425 chosen_base_url = None426 427 if base_url != METADATA_BASE_URL:428 chosen_base_url = base_url429 elif custom_metadata_endpoint:430 chosen_base_url = custom_metadata_endpoint431 elif requires_ipv6:432 chosen_base_url = METADATA_BASE_URL_IPv6433 else:434 chosen_base_url = METADATA_BASE_URL435 436 logger.debug("IMDS ENDPOINT: %s" % chosen_base_url)437 if not is_valid_uri(chosen_base_url):438 raise InvalidIMDSEndpointError(endpoint=chosen_base_url)439 440 return chosen_base_url441 442 def _construct_url(self, path):443 sep = ''444 if self._base_url and not self._base_url.endswith('/'):445 sep = '/'446 return f'{self._base_url}{sep}{path}'447 448 def _fetch_metadata_token(self):449 self._assert_enabled()450 url = self._construct_url(self._TOKEN_PATH)451 headers = {452 'x-aws-ec2-metadata-token-ttl-seconds': self._TOKEN_TTL,453 }454 self._add_user_agent(headers)455 request = botocore.awsrequest.AWSRequest(456 method='PUT', url=url, headers=headers457 )458 for i in range(self._num_attempts):459 try:460 response = self._session.send(request.prepare())461 if response.status_code == 200:462 return response.text463 elif response.status_code in (404, 403, 405):464 return None465 elif response.status_code in (400,):466 raise BadIMDSRequestError(request)467 except ReadTimeoutError:468 return None469 except RETRYABLE_HTTP_ERRORS as e:470 logger.debug(471 "Caught retryable HTTP exception while making metadata "472 "service request to %s: %s",473 url,474 e,475 exc_info=True,476 )477 except HTTPClientError as e:478 if isinstance(e.kwargs.get('error'), LocationParseError):479 raise InvalidIMDSEndpointError(endpoint=url, error=e)480 else:481 raise482 return None483 484 def _get_request(self, url_path, retry_func, token=None):485 """Make a get request to the Instance Metadata Service.486 487 :type url_path: str488 :param url_path: The path component of the URL to make a get request.489 This arg is appended to the base_url that was provided in the490 initializer.491 492 :type retry_func: callable493 :param retry_func: A function that takes the response as an argument494 and determines if it needs to retry. By default empty and non495 200 OK responses are retried.496 497 :type token: str498 :param token: Metadata token to send along with GET requests to IMDS.499 """500 self._assert_enabled()501 if not token:502 self._assert_v1_enabled()503 if retry_func is None:504 retry_func = self._default_retry505 url = self._construct_url(url_path)506 headers = {}507 if token is not None:508 headers['x-aws-ec2-metadata-token'] = token509 self._add_user_agent(headers)510 for i in range(self._num_attempts):511 try:512 request = botocore.awsrequest.AWSRequest(513 method='GET', url=url, headers=headers514 )515 response = self._session.send(request.prepare())516 if not retry_func(response):517 return response518 except RETRYABLE_HTTP_ERRORS as e:519 logger.debug(520 "Caught retryable HTTP exception while making metadata "521 "service request to %s: %s",522 url,523 e,524 exc_info=True,525 )526 raise self._RETRIES_EXCEEDED_ERROR_CLS()527 528 def _add_user_agent(self, headers):529 if self._user_agent is not None:530 headers['User-Agent'] = self._user_agent531 532 def _assert_enabled(self):533 if self._disabled:534 logger.debug("Access to EC2 metadata has been disabled.")535 raise self._RETRIES_EXCEEDED_ERROR_CLS()536 537 def _assert_v1_enabled(self):538 if self._imds_v1_disabled:539 raise MetadataRetrievalError(540 error_msg="Unable to retrieve token for use in IMDSv2 call and IMDSv1 has been disabled"541 )542 543 def _default_retry(self, response):544 return self._is_non_ok_response(response) or self._is_empty(response)545 546 def _is_non_ok_response(self, response):547 if response.status_code != 200:548 self._log_imds_response(response, 'non-200', log_body=True)549 return True550 return False551 552 def _is_empty(self, response):553 if not response.content:554 self._log_imds_response(response, 'no body', log_body=True)555 return True556 return False557 558 def _log_imds_response(self, response, reason_to_log, log_body=False):559 statement = (560 "Metadata service returned %s response "561 "with status code of %s for url: %s"562 )563 logger_args = [reason_to_log, response.status_code, response.url]564 if log_body:565 statement += ", content body: %s"566 logger_args.append(response.content)567 logger.debug(statement, *logger_args)568 569 570class InstanceMetadataFetcher(IMDSFetcher):571 _URL_PATH = 'latest/meta-data/iam/security-credentials/'572 _REQUIRED_CREDENTIAL_FIELDS = [573 'AccessKeyId',574 'SecretAccessKey',575 'Token',576 'Expiration',577 ]578 579 def retrieve_iam_role_credentials(self):580 try:581 token = self._fetch_metadata_token()582 role_name = self._get_iam_role(token)583 credentials = self._get_credentials(role_name, token)584 if self._contains_all_credential_fields(credentials):585 credentials = {586 'role_name': role_name,587 'access_key': credentials['AccessKeyId'],588 'secret_key': credentials['SecretAccessKey'],589 'token': credentials['Token'],590 'expiry_time': credentials['Expiration'],591 }592 self._evaluate_expiration(credentials)593 return credentials594 else:595 # IMDS can return a 200 response that has a JSON formatted596 # error message (i.e. if ec2 is not trusted entity for the597 # attached role). We do not necessarily want to retry for598 # these and we also do not necessarily want to raise a key599 # error. So at least log the problematic response and return600 # an empty dictionary to signal that it was not able to601 # retrieve credentials. These error will contain both a602 # Code and Message key.603 if 'Code' in credentials and 'Message' in credentials:604 logger.debug(605 'Error response received when retrieving'606 'credentials: %s.',607 credentials,608 )609 return {}610 except self._RETRIES_EXCEEDED_ERROR_CLS:611 logger.debug(612 "Max number of attempts exceeded (%s) when "613 "attempting to retrieve data from metadata service.",614 self._num_attempts,615 )616 except BadIMDSRequestError as e:617 logger.debug("Bad IMDS request: %s", e.request)618 return {}619 620 def _get_iam_role(self, token=None):621 return self._get_request(622 url_path=self._URL_PATH,623 retry_func=self._needs_retry_for_role_name,624 token=token,625 ).text626 627 def _get_credentials(self, role_name, token=None):628 r = self._get_request(629 url_path=self._URL_PATH + role_name,630 retry_func=self._needs_retry_for_credentials,631 token=token,632 )633 return json.loads(r.text)634 635 def _is_invalid_json(self, response):636 try:637 json.loads(response.text)638 return False639 except ValueError:640 self._log_imds_response(response, 'invalid json')641 return True642 643 def _needs_retry_for_role_name(self, response):644 return self._is_non_ok_response(response) or self._is_empty(response)645 646 def _needs_retry_for_credentials(self, response):647 return (648 self._is_non_ok_response(response)649 or self._is_empty(response)650 or self._is_invalid_json(response)651 )652 653 def _contains_all_credential_fields(self, credentials):654 for field in self._REQUIRED_CREDENTIAL_FIELDS:655 if field not in credentials:656 logger.debug(657 'Retrieved credentials is missing required field: %s',658 field,659 )660 return False661 return True662 663 def _evaluate_expiration(self, credentials):664 expiration = credentials.get("expiry_time")665 if expiration is None:666 return667 try:668 expiration = datetime.datetime.strptime(669 expiration, "%Y-%m-%dT%H:%M:%SZ"670 )671 refresh_interval = self._config.get(672 "ec2_credential_refresh_window", 60 * 10673 )674 jitter = random.randint(120, 600) # Between 2 to 10 minutes675 refresh_interval_with_jitter = refresh_interval + jitter676 current_time = datetime.datetime.utcnow()677 refresh_offset = datetime.timedelta(678 seconds=refresh_interval_with_jitter679 )680 extension_time = expiration - refresh_offset681 if current_time >= extension_time:682 new_time = current_time + refresh_offset683 credentials["expiry_time"] = new_time.strftime(684 "%Y-%m-%dT%H:%M:%SZ"685 )686 logger.info(687 f"Attempting credential expiration extension due to a "688 f"credential service availability issue. A refresh of "689 f"these credentials will be attempted again within "690 f"the next {refresh_interval_with_jitter/60:.0f} minutes."691 )692 except ValueError:693 logger.debug(694 f"Unable to parse expiry_time in {credentials['expiry_time']}"695 )696 697 698class IMDSRegionProvider:699 def __init__(self, session, environ=None, fetcher=None):700 """Initialize IMDSRegionProvider.701 :type session: :class:`botocore.session.Session`702 :param session: The session is needed to look up configuration for703 how to contact the instance metadata service. Specifically the704 whether or not it should use the IMDS region at all, and if so how705 to configure the timeout and number of attempts to reach the706 service.707 :type environ: None or dict708 :param environ: A dictionary of environment variables to use. If709 ``None`` is the argument then ``os.environ`` will be used by710 default.711 :type fecther: :class:`botocore.utils.InstanceMetadataRegionFetcher`712 :param fetcher: The class to actually handle the fetching of the region713 from the IMDS. If not provided a default one will be created.714 """715 self._session = session716 if environ is None:717 environ = os.environ718 self._environ = environ719 self._fetcher = fetcher720 721 def provide(self):722 """Provide the region value from IMDS."""723 instance_region = self._get_instance_metadata_region()724 return instance_region725 726 def _get_instance_metadata_region(self):727 fetcher = self._get_fetcher()728 region = fetcher.retrieve_region()729 return region730 731 def _get_fetcher(self):732 if self._fetcher is None:733 self._fetcher = self._create_fetcher()734 return self._fetcher735 736 def _create_fetcher(self):737 metadata_timeout = self._session.get_config_variable(738 'metadata_service_timeout'739 )740 metadata_num_attempts = self._session.get_config_variable(741 'metadata_service_num_attempts'742 )743 imds_config = {744 'ec2_metadata_service_endpoint': self._session.get_config_variable(745 'ec2_metadata_service_endpoint'746 ),747 'ec2_metadata_service_endpoint_mode': resolve_imds_endpoint_mode(748 self._session749 ),750 'ec2_metadata_v1_disabled': self._session.get_config_variable(751 'ec2_metadata_v1_disabled'752 ),753 }754 fetcher = InstanceMetadataRegionFetcher(755 timeout=metadata_timeout,756 num_attempts=metadata_num_attempts,757 env=self._environ,758 user_agent=self._session.user_agent(),759 config=imds_config,760 )761 return fetcher762 763 764class InstanceMetadataRegionFetcher(IMDSFetcher):765 _URL_PATH = 'latest/meta-data/placement/availability-zone/'766 767 def retrieve_region(self):768 """Get the current region from the instance metadata service.769 :rvalue: str770 :returns: The region the current instance is running in or None771 if the instance metadata service cannot be contacted or does not772 give a valid response.773 :rtype: None or str774 :returns: Returns the region as a string if it is configured to use775 IMDS as a region source. Otherwise returns ``None``. It will also776 return ``None`` if it fails to get the region from IMDS due to777 exhausting its retries or not being able to connect.778 """779 try:780 region = self._get_region()781 return region782 except self._RETRIES_EXCEEDED_ERROR_CLS:783 logger.debug(784 "Max number of attempts exceeded (%s) when "785 "attempting to retrieve data from metadata service.",786 self._num_attempts,787 )788 return None789 790 def _get_region(self):791 token = self._fetch_metadata_token()792 response = self._get_request(793 url_path=self._URL_PATH,794 retry_func=self._default_retry,795 token=token,796 )797 availability_zone = response.text798 region = availability_zone[:-1]799 return region800 801 802def merge_dicts(dict1, dict2, append_lists=False):803 """Given two dict, merge the second dict into the first.804 805 The dicts can have arbitrary nesting.806 807 :param append_lists: If true, instead of clobbering a list with the new808 value, append all of the new values onto the original list.809 """810 for key in dict2:811 if isinstance(dict2[key], dict):812 if key in dict1 and key in dict2:813 merge_dicts(dict1[key], dict2[key])814 else:815 dict1[key] = dict2[key]816 # If the value is a list and the ``append_lists`` flag is set,817 # append the new values onto the original list818 elif isinstance(dict2[key], list) and append_lists:819 # The value in dict1 must be a list in order to append new820 # values onto it.821 if key in dict1 and isinstance(dict1[key], list):822 dict1[key].extend(dict2[key])823 else:824 dict1[key] = dict2[key]825 else:826 # At scalar types, we iterate and merge the827 # current dict that we're on.828 dict1[key] = dict2[key]829 830 831def lowercase_dict(original):832 """Copies the given dictionary ensuring all keys are lowercase strings."""833 copy = {}834 for key in original:835 copy[key.lower()] = original[key]836 return copy837 838 839def parse_key_val_file(filename, _open=open):840 try:841 with _open(filename) as f:842 contents = f.read()843 return parse_key_val_file_contents(contents)844 except OSError:845 raise ConfigNotFound(path=filename)846 847 848def parse_key_val_file_contents(contents):849 # This was originally extracted from the EC2 credential provider, which was850 # fairly lenient in its parsing. We only try to parse key/val pairs if851 # there's a '=' in the line.852 final = {}853 for line in contents.splitlines():854 if '=' not in line:855 continue856 key, val = line.split('=', 1)857 key = key.strip()858 val = val.strip()859 final[key] = val860 return final861 862 863def percent_encode_sequence(mapping, safe=SAFE_CHARS):864 """Urlencode a dict or list into a string.865 866 This is similar to urllib.urlencode except that:867 868 * It uses quote, and not quote_plus869 * It has a default list of safe chars that don't need870 to be encoded, which matches what AWS services expect.871 872 If any value in the input ``mapping`` is a list type,873 then each list element wil be serialized. This is the equivalent874 to ``urlencode``'s ``doseq=True`` argument.875 876 This function should be preferred over the stdlib877 ``urlencode()`` function.878 879 :param mapping: Either a dict to urlencode or a list of880 ``(key, value)`` pairs.881 882 """883 encoded_pairs = []884 if hasattr(mapping, 'items'):885 pairs = mapping.items()886 else:887 pairs = mapping888 for key, value in pairs:889 if isinstance(value, list):890 for element in value:891 encoded_pairs.append(892 f'{percent_encode(key)}={percent_encode(element)}'893 )894 else:895 encoded_pairs.append(896 f'{percent_encode(key)}={percent_encode(value)}'897 )898 return '&'.join(encoded_pairs)899 900 901def percent_encode(input_str, safe=SAFE_CHARS):902 """Urlencodes a string.903 904 Whereas percent_encode_sequence handles taking a dict/sequence and905 producing a percent encoded string, this function deals only with906 taking a string (not a dict/sequence) and percent encoding it.907 908 If given the binary type, will simply URL encode it. If given the909 text type, will produce the binary type by UTF-8 encoding the910 text. If given something else, will convert it to the text type911 first.912 """913 # If its not a binary or text string, make it a text string.914 if not isinstance(input_str, (bytes, str)):915 input_str = str(input_str)916 # If it's not bytes, make it bytes by UTF-8 encoding it.917 if not isinstance(input_str, bytes):918 input_str = input_str.encode('utf-8')919 return quote(input_str, safe=safe)920 921 922def _epoch_seconds_to_datetime(value, tzinfo):923 """Parse numerical epoch timestamps (seconds since 1970) into a924 ``datetime.datetime`` in UTC using ``datetime.timedelta``. This is intended925 as fallback when ``fromtimestamp`` raises ``OverflowError`` or ``OSError``.926 927 :type value: float or int928 :param value: The Unix timestamps as number.929 930 :type tzinfo: callable931 :param tzinfo: A ``datetime.tzinfo`` class or compatible callable.932 """933 epoch_zero = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())934 epoch_zero_localized = epoch_zero.astimezone(tzinfo())935 return epoch_zero_localized + datetime.timedelta(seconds=value)936 937 938def _parse_timestamp_with_tzinfo(value, tzinfo):939 """Parse timestamp with pluggable tzinfo options."""940 if isinstance(value, (int, float)):941 # Possibly an epoch time.942 return datetime.datetime.fromtimestamp(value, tzinfo())943 else:944 try:945 return datetime.datetime.fromtimestamp(float(value), tzinfo())946 except (TypeError, ValueError):947 pass948 try:949 # In certain cases, a timestamp marked with GMT can be parsed into a950 # different time zone, so here we provide a context which will951 # enforce that GMT == UTC.952 return dateutil.parser.parse(value, tzinfos={'GMT': tzutc()})953 except (TypeError, ValueError) as e:954 raise ValueError(f'Invalid timestamp "{value}": {e}')955 956 957def parse_timestamp(value):958 """Parse a timestamp into a datetime object.959 960 Supported formats:961 962 * iso8601963 * rfc822964 * epoch (value is an integer)965 966 This will return a ``datetime.datetime`` object.967 968 """969 tzinfo_options = get_tzinfo_options()970 for tzinfo in tzinfo_options:971 try:972 return _parse_timestamp_with_tzinfo(value, tzinfo)973 except (OSError, OverflowError) as e:974 logger.debug(975 'Unable to parse timestamp with "%s" timezone info.',976 tzinfo.__name__,977 exc_info=e,978 )979 # For numeric values attempt fallback to using fromtimestamp-free method.980 # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This981 # may raise ``OverflowError``, if the timestamp is out of the range of982 # values supported by the platform C localtime() function, and ``OSError``983 # on localtime() failure. It's common for this to be restricted to years984 # from 1970 through 2038."985 try:986 numeric_value = float(value)987 except (TypeError, ValueError):988 pass989 else:990 try:991 for tzinfo in tzinfo_options:992 return _epoch_seconds_to_datetime(numeric_value, tzinfo=tzinfo)993 except (OSError, OverflowError) as e:994 logger.debug(995 'Unable to parse timestamp using fallback method with "%s" '996 'timezone info.',997 tzinfo.__name__,998 exc_info=e,999 )1000 raise RuntimeError(1001 'Unable to calculate correct timezone offset for "%s"' % value1002 )1003 1004 1005def parse_to_aware_datetime(value):1006 """Converted the passed in value to a datetime object with tzinfo.1007 1008 This function can be used to normalize all timestamp inputs. This1009 function accepts a number of different types of inputs, but1010 will always return a datetime.datetime object with time zone1011 information.1012 1013 The input param ``value`` can be one of several types:1014 1015 * A datetime object (both naive and aware)1016 * An integer representing the epoch time (can also be a string1017 of the integer, i.e '0', instead of 0). The epoch time is1018 considered to be UTC.1019 * An iso8601 formatted timestamp. This does not need to be1020 a complete timestamp, it can contain just the date portion1021 without the time component.1022 1023 The returned value will be a datetime object that will have tzinfo.1024 If no timezone info was provided in the input value, then UTC is1025 assumed, not local time.1026 1027 """1028 # This is a general purpose method that handles several cases of1029 # converting the provided value to a string timestamp suitable to be1030 # serialized to an http request. It can handle:1031 # 1) A datetime.datetime object.1032 if isinstance(value, _DatetimeClass):1033 datetime_obj = value1034 else:1035 # 2) A string object that's formatted as a timestamp.1036 # We document this as being an iso8601 timestamp, although1037 # parse_timestamp is a bit more flexible.1038 datetime_obj = parse_timestamp(value)1039 if datetime_obj.tzinfo is None:1040 # I think a case would be made that if no time zone is provided,1041 # we should use the local time. However, to restore backwards1042 # compat, the previous behavior was to assume UTC, which is1043 # what we're going to do here.1044 datetime_obj = datetime_obj.replace(tzinfo=tzutc())1045 else:1046 datetime_obj = datetime_obj.astimezone(tzutc())1047 return datetime_obj1048 1049 1050def datetime2timestamp(dt, default_timezone=None):1051 """Calculate the timestamp based on the given datetime instance.1052 1053 :type dt: datetime1054 :param dt: A datetime object to be converted into timestamp1055 :type default_timezone: tzinfo1056 :param default_timezone: If it is provided as None, we treat it as tzutc().1057 But it is only used when dt is a naive datetime.1058 :returns: The timestamp1059 """1060 epoch = datetime.datetime(1970, 1, 1)1061 if dt.tzinfo is None:1062 if default_timezone is None:1063 default_timezone = tzutc()1064 dt = dt.replace(tzinfo=default_timezone)1065 d = dt.replace(tzinfo=None) - dt.utcoffset() - epoch1066 if hasattr(d, "total_seconds"):1067 return d.total_seconds() # Works in Python 3.6+1068 return (1069 d.microseconds + (d.seconds + d.days * 24 * 3600) * 10**61070 ) / 10**61071 1072 1073def calculate_sha256(body, as_hex=False):1074 """Calculate a sha256 checksum.1075 1076 This method will calculate the sha256 checksum of a file like1077 object. Note that this method will iterate through the entire1078 file contents. The caller is responsible for ensuring the proper1079 starting position of the file and ``seek()``'ing the file back1080 to its starting location if other consumers need to read from1081 the file like object.1082 1083 :param body: Any file like object. The file must be opened1084 in binary mode such that a ``.read()`` call returns bytes.1085 :param as_hex: If True, then the hex digest is returned.1086 If False, then the digest (as binary bytes) is returned.1087 1088 :returns: The sha256 checksum1089 1090 """1091 checksum = hashlib.sha256()1092 for chunk in iter(lambda: body.read(1024 * 1024), b''):1093 checksum.update(chunk)1094 if as_hex:1095 return checksum.hexdigest()1096 else:1097 return checksum.digest()1098 1099 1100def calculate_tree_hash(body):1101 """Calculate a tree hash checksum.1102 1103 For more information see:1104 1105 http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html1106 1107 :param body: Any file like object. This has the same constraints as1108 the ``body`` param in calculate_sha2561109 1110 :rtype: str1111 :returns: The hex version of the calculated tree hash1112 1113 """1114 chunks = []1115 required_chunk_size = 1024 * 10241116 sha256 = hashlib.sha2561117 for chunk in iter(lambda: body.read(required_chunk_size), b''):1118 chunks.append(sha256(chunk).digest())1119 if not chunks:1120 return sha256(b'').hexdigest()1121 while len(chunks) > 1:1122 new_chunks = []1123 for first, second in _in_pairs(chunks):1124 if second is not None:1125 new_chunks.append(sha256(first + second).digest())1126 else:1127 # We're at the end of the list and there's no pair left.1128 new_chunks.append(first)1129 chunks = new_chunks1130 return binascii.hexlify(chunks[0]).decode('ascii')1131 1132 1133def _in_pairs(iterable):1134 # Creates iterator that iterates over the list in pairs:1135 # for a, b in _in_pairs([0, 1, 2, 3, 4]):1136 # print(a, b)1137 #1138 # will print:1139 # 0, 11140 # 2, 31141 # 4, None1142 shared_iter = iter(iterable)1143 # Note that zip_longest is a compat import that uses1144 # the itertools izip_longest. This creates an iterator,1145 # this call below does _not_ immediately create the list1146 # of pairs.1147 return zip_longest(shared_iter, shared_iter)1148 1149 1150class CachedProperty:1151 """A read only property that caches the initially computed value.1152 1153 This descriptor will only call the provided ``fget`` function once.1154 Subsequent access to this property will return the cached value.1155 1156 """1157 1158 def __init__(self, fget):1159 self._fget = fget1160 1161 def __get__(self, obj, cls):1162 if obj is None:1163 return self1164 else:1165 computed_value = self._fget(obj)1166 obj.__dict__[self._fget.__name__] = computed_value1167 return computed_value1168 1169 1170class ArgumentGenerator:1171 """Generate sample input based on a shape model.1172 1173 This class contains a ``generate_skeleton`` method that will take1174 an input/output shape (created from ``botocore.model``) and generate1175 a sample dictionary corresponding to the input/output shape.1176 1177 The specific values used are place holder values. For strings either an1178 empty string or the member name can be used, for numbers 0 or 0.0 is used.1179 The intended usage of this class is to generate the *shape* of the input1180 structure.1181 1182 This can be useful for operations that have complex input shapes.1183 This allows a user to just fill in the necessary data instead of1184 worrying about the specific structure of the input arguments.1185 1186 Example usage::1187 1188 s = botocore.session.get_session()1189 ddb = s.get_service_model('dynamodb')1190 arg_gen = ArgumentGenerator()1191 sample_input = arg_gen.generate_skeleton(1192 ddb.operation_model('CreateTable').input_shape)1193 print("Sample input for dynamodb.CreateTable: %s" % sample_input)1194 1195 """1196 1197 def __init__(self, use_member_names=False):1198 self._use_member_names = use_member_names1199 1200 def generate_skeleton(self, shape):