codekingpro/portable-devtools
114k
1# Copyright 2014 Google Inc. All Rights Reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""Classes to encapsulate a single HTTP request.16 17The classes implement a command pattern, with every18object supporting an execute() method that does the19actual HTTP request.20"""21from __future__ import absolute_import22 23__author__ = "jcgregorio@google.com (Joe Gregorio)"24 25import copy26import http.client as http_client27import io28import json29import logging30import mimetypes31import os32import random33import socket34import time35import urllib36import uuid37 38import httplib239 40# TODO(issue 221): Remove this conditional import jibbajabba.41try:42 import ssl43except ImportError:44 _ssl_SSLError = object()45else:46 _ssl_SSLError = ssl.SSLError47 48from email.generator import Generator49from email.mime.multipart import MIMEMultipart50from email.mime.nonmultipart import MIMENonMultipart51from email.parser import FeedParser52 53from googleapiclient import _auth54from googleapiclient import _helpers as util55from googleapiclient.errors import (56 BatchError,57 HttpError,58 InvalidChunkSizeError,59 ResumableUploadError,60 UnexpectedBodyError,61 UnexpectedMethodError,62)63from googleapiclient.model import JsonModel64 65LOGGER = logging.getLogger(__name__)66 67DEFAULT_CHUNK_SIZE = 100 * 1024 * 102468 69MAX_URI_LENGTH = 204870 71MAX_BATCH_LIMIT = 100072 73_TOO_MANY_REQUESTS = 42974 75DEFAULT_HTTP_TIMEOUT_SEC = 6076 77_LEGACY_BATCH_URI = "https://www.googleapis.com/batch"78 79 80def _should_retry_response(resp_status, content):81 """Determines whether a response should be retried.82 83 Args:84 resp_status: The response status received.85 content: The response content body.86 87 Returns:88 True if the response should be retried, otherwise False.89 """90 reason = None91 92 # Retry on 5xx errors.93 if resp_status >= 500:94 return True95 96 # Retry on 429 errors.97 if resp_status == _TOO_MANY_REQUESTS:98 return True99 100 # For 403 errors, we have to check for the `reason` in the response to101 # determine if we should retry.102 if resp_status == http_client.FORBIDDEN:103 # If there's no details about the 403 type, don't retry.104 if not content:105 return False106 107 # Content is in JSON format.108 try:109 data = json.loads(content.decode("utf-8"))110 if isinstance(data, dict):111 # There are many variations of the error json so we need112 # to determine the keyword which has the error detail. Make sure113 # that the order of the keywords below isn't changed as it can114 # break user code. If the "errors" key exists, we must use that115 # first.116 # See Issue #1243117 # https://github.com/googleapis/google-api-python-client/issues/1243118 error_detail_keyword = next(119 (120 kw121 for kw in ["errors", "status", "message"]122 if kw in data["error"]123 ),124 "",125 )126 127 if error_detail_keyword:128 reason = data["error"][error_detail_keyword]129 130 if isinstance(reason, list) and len(reason) > 0:131 reason = reason[0]132 if "reason" in reason:133 reason = reason["reason"]134 else:135 reason = data[0]["error"]["errors"]["reason"]136 except (UnicodeDecodeError, ValueError, KeyError):137 LOGGER.warning("Invalid JSON content from response: %s", content)138 return False139 140 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)141 142 # Only retry on rate limit related failures.143 if reason in ("userRateLimitExceeded", "rateLimitExceeded"):144 return True145 146 # Everything else is a success or non-retriable so break.147 return False148 149 150def _retry_request(151 http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs152):153 """Retries an HTTP request multiple times while handling errors.154 155 If after all retries the request still fails, last error is either returned as156 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).157 158 Args:159 http: Http object to be used to execute request.160 num_retries: Maximum number of retries.161 req_type: Type of the request (used for logging retries).162 sleep, rand: Functions to sleep for random time between retries.163 uri: URI to be requested.164 method: HTTP method to be used.165 args, kwargs: Additional arguments passed to http.request.166 167 Returns:168 resp, content - Response from the http request (may be HTTP 5xx).169 """170 resp = None171 content = None172 exception = None173 for retry_num in range(num_retries + 1):174 if retry_num > 0:175 # Sleep before retrying.176 sleep_time = rand() * 2**retry_num177 LOGGER.warning(178 "Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s",179 sleep_time,180 retry_num,181 num_retries,182 req_type,183 method,184 uri,185 resp.status if resp else exception,186 )187 sleep(sleep_time)188 189 try:190 exception = None191 resp, content = http.request(uri, method, *args, **kwargs)192 # Retry on SSL errors and socket timeout errors.193 except _ssl_SSLError as ssl_error:194 exception = ssl_error195 except socket.timeout as socket_timeout:196 # Needs to be before socket.error as it's a subclass of OSError197 # socket.timeout has no errorcode198 exception = socket_timeout199 except ConnectionError as connection_error:200 # Needs to be before socket.error as it's a subclass of OSError201 exception = connection_error202 except OSError as socket_error:203 # errno's contents differ by platform, so we have to match by name.204 # Some of these same errors may have been caught above, e.g. ECONNRESET *should* be205 # raised as a ConnectionError, but some libraries will raise it as a socket.error206 # with an errno corresponding to ECONNRESET207 if socket.errno.errorcode.get(socket_error.errno) not in {208 "WSAETIMEDOUT",209 "ETIMEDOUT",210 "EPIPE",211 "ECONNABORTED",212 "ECONNREFUSED",213 "ECONNRESET",214 }:215 raise216 exception = socket_error217 except httplib2.ServerNotFoundError as server_not_found_error:218 exception = server_not_found_error219 220 if exception:221 if retry_num == num_retries:222 raise exception223 else:224 continue225 226 if not _should_retry_response(resp.status, content):227 break228 229 return resp, content230 231 232class MediaUploadProgress(object):233 """Status of a resumable upload."""234 235 def __init__(self, resumable_progress, total_size):236 """Constructor.237 238 Args:239 resumable_progress: int, bytes sent so far.240 total_size: int, total bytes in complete upload, or None if the total241 upload size isn't known ahead of time.242 """243 self.resumable_progress = resumable_progress244 self.total_size = total_size245 246 def progress(self):247 """Percent of upload completed, as a float.248 249 Returns:250 the percentage complete as a float, returning 0.0 if the total size of251 the upload is unknown.252 """253 if self.total_size is not None and self.total_size != 0:254 return float(self.resumable_progress) / float(self.total_size)255 else:256 return 0.0257 258 259class MediaDownloadProgress(object):260 """Status of a resumable download."""261 262 def __init__(self, resumable_progress, total_size):263 """Constructor.264 265 Args:266 resumable_progress: int, bytes received so far.267 total_size: int, total bytes in complete download.268 """269 self.resumable_progress = resumable_progress270 self.total_size = total_size271 272 def progress(self):273 """Percent of download completed, as a float.274 275 Returns:276 the percentage complete as a float, returning 0.0 if the total size of277 the download is unknown.278 """279 if self.total_size is not None and self.total_size != 0:280 return float(self.resumable_progress) / float(self.total_size)281 else:282 return 0.0283 284 285class MediaUpload(object):286 """Describes a media object to upload.287 288 Base class that defines the interface of MediaUpload subclasses.289 290 Note that subclasses of MediaUpload may allow you to control the chunksize291 when uploading a media object. It is important to keep the size of the chunk292 as large as possible to keep the upload efficient. Other factors may influence293 the size of the chunk you use, particularly if you are working in an294 environment where individual HTTP requests may have a hardcoded time limit,295 such as under certain classes of requests under Google App Engine.296 297 Streams are io.Base compatible objects that support seek(). Some MediaUpload298 subclasses support using streams directly to upload data. Support for299 streaming may be indicated by a MediaUpload sub-class and if appropriate for a300 platform that stream will be used for uploading the media object. The support301 for streaming is indicated by has_stream() returning True. The stream() method302 should return an io.Base object that supports seek(). On platforms where the303 underlying httplib module supports streaming, for example Python 2.6 and304 later, the stream will be passed into the http library which will result in305 less memory being used and possibly faster uploads.306 307 If you need to upload media that can't be uploaded using any of the existing308 MediaUpload sub-class then you can sub-class MediaUpload for your particular309 needs.310 """311 312 def chunksize(self):313 """Chunk size for resumable uploads.314 315 Returns:316 Chunk size in bytes.317 """318 raise NotImplementedError()319 320 def mimetype(self):321 """Mime type of the body.322 323 Returns:324 Mime type.325 """326 return "application/octet-stream"327 328 def size(self):329 """Size of upload.330 331 Returns:332 Size of the body, or None of the size is unknown.333 """334 return None335 336 def resumable(self):337 """Whether this upload is resumable.338 339 Returns:340 True if resumable upload or False.341 """342 return False343 344 def getbytes(self, begin, end):345 """Get bytes from the media.346 347 Args:348 begin: int, offset from beginning of file.349 length: int, number of bytes to read, starting at begin.350 351 Returns:352 A string of bytes read. May be shorter than length if EOF was reached353 first.354 """355 raise NotImplementedError()356 357 def has_stream(self):358 """Does the underlying upload support a streaming interface.359 360 Streaming means it is an io.IOBase subclass that supports seek, i.e.361 seekable() returns True.362 363 Returns:364 True if the call to stream() will return an instance of a seekable io.Base365 subclass.366 """367 return False368 369 def stream(self):370 """A stream interface to the data being uploaded.371 372 Returns:373 The returned value is an io.IOBase subclass that supports seek, i.e.374 seekable() returns True.375 """376 raise NotImplementedError()377 378 @util.positional(1)379 def _to_json(self, strip=None):380 """Utility function for creating a JSON representation of a MediaUpload.381 382 Args:383 strip: array, An array of names of members to not include in the JSON.384 385 Returns:386 string, a JSON representation of this instance, suitable to pass to387 from_json().388 """389 t = type(self)390 d = copy.copy(self.__dict__)391 if strip is not None:392 for member in strip:393 del d[member]394 d["_class"] = t.__name__395 d["_module"] = t.__module__396 return json.dumps(d)397 398 def to_json(self):399 """Create a JSON representation of an instance of MediaUpload.400 401 Returns:402 string, a JSON representation of this instance, suitable to pass to403 from_json().404 """405 return self._to_json()406 407 @classmethod408 def new_from_json(cls, s):409 """Utility class method to instantiate a MediaUpload subclass from a JSON410 representation produced by to_json().411 412 Args:413 s: string, JSON from to_json().414 415 Returns:416 An instance of the subclass of MediaUpload that was serialized with417 to_json().418 """419 data = json.loads(s)420 # Find and call the right classmethod from_json() to restore the object.421 module = data["_module"]422 m = __import__(module, fromlist=module.split(".")[:-1])423 kls = getattr(m, data["_class"])424 from_json = getattr(kls, "from_json")425 return from_json(s)426 427 428class MediaIoBaseUpload(MediaUpload):429 """A MediaUpload for a io.Base objects.430 431 Note that the Python file object is compatible with io.Base and can be used432 with this class also.433 434 fh = BytesIO('...Some data to upload...')435 media = MediaIoBaseUpload(fh, mimetype='image/png',436 chunksize=1024*1024, resumable=True)437 farm.animals().insert(438 id='cow',439 name='cow.png',440 media_body=media).execute()441 442 Depending on the platform you are working on, you may pass -1 as the443 chunksize, which indicates that the entire file should be uploaded in a single444 request. If the underlying platform supports streams, such as Python 2.6 or445 later, then this can be very efficient as it avoids multiple connections, and446 also avoids loading the entire file into memory before sending it. Note that447 Google App Engine has a 5MB limit on request size, so you should never set448 your chunksize larger than 5MB, or to -1.449 """450 451 @util.positional(3)452 def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):453 """Constructor.454 455 Args:456 fd: io.Base or file object, The source of the bytes to upload. MUST be457 opened in blocking mode, do not use streams opened in non-blocking mode.458 The given stream must be seekable, that is, it must be able to call459 seek() on fd.460 mimetype: string, Mime-type of the file.461 chunksize: int, File will be uploaded in chunks of this many bytes. Only462 used if resumable=True. Pass in a value of -1 if the file is to be463 uploaded as a single chunk. Note that Google App Engine has a 5MB limit464 on request size, so you should never set your chunksize larger than 5MB,465 or to -1.466 resumable: bool, True if this is a resumable upload. False means upload467 in a single request.468 """469 super(MediaIoBaseUpload, self).__init__()470 self._fd = fd471 self._mimetype = mimetype472 if not (chunksize == -1 or chunksize > 0):473 raise InvalidChunkSizeError()474 self._chunksize = chunksize475 self._resumable = resumable476 477 self._fd.seek(0, os.SEEK_END)478 self._size = self._fd.tell()479 480 def chunksize(self):481 """Chunk size for resumable uploads.482 483 Returns:484 Chunk size in bytes.485 """486 return self._chunksize487 488 def mimetype(self):489 """Mime type of the body.490 491 Returns:492 Mime type.493 """494 return self._mimetype495 496 def size(self):497 """Size of upload.498 499 Returns:500 Size of the body, or None of the size is unknown.501 """502 return self._size503 504 def resumable(self):505 """Whether this upload is resumable.506 507 Returns:508 True if resumable upload or False.509 """510 return self._resumable511 512 def getbytes(self, begin, length):513 """Get bytes from the media.514 515 Args:516 begin: int, offset from beginning of file.517 length: int, number of bytes to read, starting at begin.518 519 Returns:520 A string of bytes read. May be shorted than length if EOF was reached521 first.522 """523 self._fd.seek(begin)524 return self._fd.read(length)525 526 def has_stream(self):527 """Does the underlying upload support a streaming interface.528 529 Streaming means it is an io.IOBase subclass that supports seek, i.e.530 seekable() returns True.531 532 Returns:533 True if the call to stream() will return an instance of a seekable io.Base534 subclass.535 """536 return True537 538 def stream(self):539 """A stream interface to the data being uploaded.540 541 Returns:542 The returned value is an io.IOBase subclass that supports seek, i.e.543 seekable() returns True.544 """545 return self._fd546 547 def to_json(self):548 """This upload type is not serializable."""549 raise NotImplementedError("MediaIoBaseUpload is not serializable.")550 551 552class MediaFileUpload(MediaIoBaseUpload):553 """A MediaUpload for a file.554 555 Construct a MediaFileUpload and pass as the media_body parameter of the556 method. For example, if we had a service that allowed uploading images:557 558 media = MediaFileUpload('cow.png', mimetype='image/png',559 chunksize=1024*1024, resumable=True)560 farm.animals().insert(561 id='cow',562 name='cow.png',563 media_body=media).execute()564 565 Depending on the platform you are working on, you may pass -1 as the566 chunksize, which indicates that the entire file should be uploaded in a single567 request. If the underlying platform supports streams, such as Python 2.6 or568 later, then this can be very efficient as it avoids multiple connections, and569 also avoids loading the entire file into memory before sending it. Note that570 Google App Engine has a 5MB limit on request size, so you should never set571 your chunksize larger than 5MB, or to -1.572 """573 574 @util.positional(2)575 def __init__(576 self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False577 ):578 """Constructor.579 580 Args:581 filename: string, Name of the file.582 mimetype: string, Mime-type of the file. If None then a mime-type will be583 guessed from the file extension.584 chunksize: int, File will be uploaded in chunks of this many bytes. Only585 used if resumable=True. Pass in a value of -1 if the file is to be586 uploaded in a single chunk. Note that Google App Engine has a 5MB limit587 on request size, so you should never set your chunksize larger than 5MB,588 or to -1.589 resumable: bool, True if this is a resumable upload. False means upload590 in a single request.591 """592 self._fd = None593 self._filename = filename594 self._fd = open(self._filename, "rb")595 if mimetype is None:596 # No mimetype provided, make a guess.597 mimetype, _ = mimetypes.guess_type(filename)598 if mimetype is None:599 # Guess failed, use octet-stream.600 mimetype = "application/octet-stream"601 super(MediaFileUpload, self).__init__(602 self._fd, mimetype, chunksize=chunksize, resumable=resumable603 )604 605 def __del__(self):606 if self._fd:607 self._fd.close()608 609 def to_json(self):610 """Creating a JSON representation of an instance of MediaFileUpload.611 612 Returns:613 string, a JSON representation of this instance, suitable to pass to614 from_json().615 """616 return self._to_json(strip=["_fd"])617 618 @staticmethod619 def from_json(s):620 d = json.loads(s)621 return MediaFileUpload(622 d["_filename"],623 mimetype=d["_mimetype"],624 chunksize=d["_chunksize"],625 resumable=d["_resumable"],626 )627 628 629class MediaInMemoryUpload(MediaIoBaseUpload):630 """MediaUpload for a chunk of bytes.631 632 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or io.StringIO for633 the stream.634 """635 636 @util.positional(2)637 def __init__(638 self,639 body,640 mimetype="application/octet-stream",641 chunksize=DEFAULT_CHUNK_SIZE,642 resumable=False,643 ):644 """Create a new MediaInMemoryUpload.645 646 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or io.StringIO for647 the stream.648 649 Args:650 body: string, Bytes of body content.651 mimetype: string, Mime-type of the file or default of652 'application/octet-stream'.653 chunksize: int, File will be uploaded in chunks of this many bytes. Only654 used if resumable=True.655 resumable: bool, True if this is a resumable upload. False means upload656 in a single request.657 """658 fd = io.BytesIO(body)659 super(MediaInMemoryUpload, self).__init__(660 fd, mimetype, chunksize=chunksize, resumable=resumable661 )662 663 664class MediaIoBaseDownload(object):665 """ "Download media resources.666 667 Note that the Python file object is compatible with io.Base and can be used668 with this class also.669 670 671 Example:672 request = farms.animals().get_media(id='cow')673 fh = io.FileIO('cow.png', mode='wb')674 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)675 676 done = False677 while done is False:678 status, done = downloader.next_chunk()679 if status:680 print "Download %d%%." % int(status.progress() * 100)681 print "Download Complete!"682 """683 684 @util.positional(3)685 def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):686 """Constructor.687 688 Args:689 fd: io.Base or file object, The stream in which to write the downloaded690 bytes.691 request: googleapiclient.http.HttpRequest, the media request to perform in692 chunks.693 chunksize: int, File will be downloaded in chunks of this many bytes.694 """695 self._fd = fd696 self._request = request697 self._uri = request.uri698 self._chunksize = chunksize699 self._progress = 0700 self._total_size = None701 self._done = False702 703 # Stubs for testing.704 self._sleep = time.sleep705 self._rand = random.random706 707 self._headers = {}708 for k, v in request.headers.items():709 # allow users to supply custom headers by setting them on the request710 # but strip out the ones that are set by default on requests generated by711 # API methods like Drive's files().get(fileId=...)712 if not k.lower() in ("accept", "accept-encoding", "user-agent"):713 self._headers[k] = v714 715 @util.positional(1)716 def next_chunk(self, num_retries=0):717 """Get the next chunk of the download.718 719 Args:720 num_retries: Integer, number of times to retry with randomized721 exponential backoff. If all retries fail, the raised HttpError722 represents the last request. If zero (default), we attempt the723 request only once.724 725 Returns:726 (status, done): (MediaDownloadProgress, boolean)727 The value of 'done' will be True when the media has been fully728 downloaded or the total size of the media is unknown.729 730 Raises:731 googleapiclient.errors.HttpError if the response was not a 2xx.732 httplib2.HttpLib2Error if a transport error has occurred.733 """734 headers = self._headers.copy()735 headers["range"] = "bytes=%d-%d" % (736 self._progress,737 self._progress + self._chunksize - 1,738 )739 http = self._request.http740 741 resp, content = _retry_request(742 http,743 num_retries,744 "media download",745 self._sleep,746 self._rand,747 self._uri,748 "GET",749 headers=headers,750 )751 752 if resp.status in [200, 206]:753 if "content-location" in resp and resp["content-location"] != self._uri:754 self._uri = resp["content-location"]755 self._progress += len(content)756 self._fd.write(content)757 758 if "content-range" in resp:759 content_range = resp["content-range"]760 length = content_range.rsplit("/", 1)[1]761 self._total_size = int(length)762 elif "content-length" in resp:763 self._total_size = int(resp["content-length"])764 765 if self._total_size is None or self._progress == self._total_size:766 self._done = True767 return MediaDownloadProgress(self._progress, self._total_size), self._done768 elif resp.status == 416:769 # 416 is Range Not Satisfiable770 # This typically occurs with a zero byte file771 content_range = resp["content-range"]772 length = content_range.rsplit("/", 1)[1]773 self._total_size = int(length)774 if self._total_size == 0:775 self._done = True776 return (777 MediaDownloadProgress(self._progress, self._total_size),778 self._done,779 )780 raise HttpError(resp, content, uri=self._uri)781 782 783class _StreamSlice(object):784 """Truncated stream.785 786 Takes a stream and presents a stream that is a slice of the original stream.787 This is used when uploading media in chunks. In later versions of Python a788 stream can be passed to httplib in place of the string of data to send. The789 problem is that httplib just blindly reads to the end of the stream. This790 wrapper presents a virtual stream that only reads to the end of the chunk.791 """792 793 def __init__(self, stream, begin, chunksize):794 """Constructor.795 796 Args:797 stream: (io.Base, file object), the stream to wrap.798 begin: int, the seek position the chunk begins at.799 chunksize: int, the size of the chunk.800 """801 self._stream = stream802 self._begin = begin803 self._chunksize = chunksize804 self._stream.seek(begin)805 806 def read(self, n=-1):807 """Read n bytes.808 809 Args:810 n, int, the number of bytes to read.811 812 Returns:813 A string of length 'n', or less if EOF is reached.814 """815 # The data left available to read sits in [cur, end)816 cur = self._stream.tell()817 end = self._begin + self._chunksize818 if n == -1 or cur + n > end:819 n = end - cur820 return self._stream.read(n)821 822 823class HttpRequest(object):824 """Encapsulates a single HTTP request."""825 826 @util.positional(4)827 def __init__(828 self,829 http,830 postproc,831 uri,832 method="GET",833 body=None,834 headers=None,835 methodId=None,836 resumable=None,837 ):838 """Constructor for an HttpRequest.839 840 Args:841 http: httplib2.Http, the transport object to use to make a request842 postproc: callable, called on the HTTP response and content to transform843 it into a data object before returning, or raising an exception844 on an error.845 uri: string, the absolute URI to send the request to846 method: string, the HTTP method to use847 body: string, the request body of the HTTP request,848 headers: dict, the HTTP request headers849 methodId: string, a unique identifier for the API method being called.850 resumable: MediaUpload, None if this is not a resumbale request.851 """852 self.uri = uri853 self.method = method854 self.body = body855 self.headers = headers or {}856 self.methodId = methodId857 self.http = http858 self.postproc = postproc859 self.resumable = resumable860 self.response_callbacks = []861 self._in_error_state = False862 863 # The size of the non-media part of the request.864 self.body_size = len(self.body or "")865 866 # The resumable URI to send chunks to.867 self.resumable_uri = None868 869 # The bytes that have been uploaded.870 self.resumable_progress = 0871 872 # Stubs for testing.873 self._rand = random.random874 self._sleep = time.sleep875 876 @util.positional(1)877 def execute(self, http=None, num_retries=0):878 """Execute the request.879 880 Args:881 http: httplib2.Http, an http object to be used in place of the882 one the HttpRequest request object was constructed with.883 num_retries: Integer, number of times to retry with randomized884 exponential backoff. If all retries fail, the raised HttpError885 represents the last request. If zero (default), we attempt the886 request only once.887 888 Returns:889 A deserialized object model of the response body as determined890 by the postproc.891 892 Raises:893 googleapiclient.errors.HttpError if the response was not a 2xx.894 httplib2.HttpLib2Error if a transport error has occurred.895 """896 if http is None:897 http = self.http898 899 if self.resumable:900 body = None901 while body is None:902 _, body = self.next_chunk(http=http, num_retries=num_retries)903 return body904 905 # Non-resumable case.906 907 if "content-length" not in self.headers:908 self.headers["content-length"] = str(self.body_size)909 # If the request URI is too long then turn it into a POST request.910 # Assume that a GET request never contains a request body.911 if len(self.uri) > MAX_URI_LENGTH and self.method == "GET":912 self.method = "POST"913 self.headers["x-http-method-override"] = "GET"914 self.headers["content-type"] = "application/x-www-form-urlencoded"915 parsed = urllib.parse.urlparse(self.uri)916 self.uri = urllib.parse.urlunparse(917 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, None)918 )919 self.body = parsed.query920 self.headers["content-length"] = str(len(self.body))921 922 # Handle retries for server-side errors.923 resp, content = _retry_request(924 http,925 num_retries,926 "request",927 self._sleep,928 self._rand,929 str(self.uri),930 method=str(self.method),931 body=self.body,932 headers=self.headers,933 )934 935 for callback in self.response_callbacks:936 callback(resp)937 if resp.status >= 300:938 raise HttpError(resp, content, uri=self.uri)939 return self.postproc(resp, content)940 941 @util.positional(2)942 def add_response_callback(self, cb):943 """add_response_headers_callback944 945 Args:946 cb: Callback to be called on receiving the response headers, of signature:947 948 def cb(resp):949 # Where resp is an instance of httplib2.Response950 """951 self.response_callbacks.append(cb)952 953 @util.positional(1)954 def next_chunk(self, http=None, num_retries=0):955 """Execute the next step of a resumable upload.956 957 Can only be used if the method being executed supports media uploads and958 the MediaUpload object passed in was flagged as using resumable upload.959 960 Example:961 962 media = MediaFileUpload('cow.png', mimetype='image/png',963 chunksize=1000, resumable=True)964 request = farm.animals().insert(965 id='cow',966 name='cow.png',967 media_body=media)968 969 response = None970 while response is None:971 status, response = request.next_chunk()972 if status:973 print "Upload %d%% complete." % int(status.progress() * 100)974 975 976 Args:977 http: httplib2.Http, an http object to be used in place of the978 one the HttpRequest request object was constructed with.979 num_retries: Integer, number of times to retry with randomized980 exponential backoff. If all retries fail, the raised HttpError981 represents the last request. If zero (default), we attempt the982 request only once.983 984 Returns:985 (status, body): (ResumableMediaStatus, object)986 The body will be None until the resumable media is fully uploaded.987 988 Raises:989 googleapiclient.errors.HttpError if the response was not a 2xx.990 httplib2.HttpLib2Error if a transport error has occurred.991 """992 if http is None:993 http = self.http994 995 if self.resumable.size() is None:996 size = "*"997 else:998 size = str(self.resumable.size())999 1000 if self.resumable_uri is None:1001 start_headers = copy.copy(self.headers)1002 start_headers["X-Upload-Content-Type"] = self.resumable.mimetype()1003 if size != "*":1004 start_headers["X-Upload-Content-Length"] = size1005 start_headers["content-length"] = str(self.body_size)1006 1007 resp, content = _retry_request(1008 http,1009 num_retries,1010 "resumable URI request",1011 self._sleep,1012 self._rand,1013 self.uri,1014 method=self.method,1015 body=self.body,1016 headers=start_headers,1017 )1018 1019 if resp.status == 200 and "location" in resp:1020 self.resumable_uri = resp["location"]1021 else:1022 raise ResumableUploadError(resp, content)1023 elif self._in_error_state:1024 # If we are in an error state then query the server for current state of1025 # the upload by sending an empty PUT and reading the 'range' header in1026 # the response.1027 headers = {"Content-Range": "bytes */%s" % size, "content-length": "0"}1028 resp, content = http.request(self.resumable_uri, "PUT", headers=headers)1029 status, body = self._process_response(resp, content)1030 if body:1031 # The upload was complete.1032 return (status, body)1033 1034 if self.resumable.has_stream():1035 data = self.resumable.stream()1036 if self.resumable.chunksize() == -1:1037 data.seek(self.resumable_progress)1038 chunk_end = self.resumable.size() - self.resumable_progress - 11039 else:1040 # Doing chunking with a stream, so wrap a slice of the stream.1041 data = _StreamSlice(1042 data, self.resumable_progress, self.resumable.chunksize()1043 )1044 chunk_end = min(1045 self.resumable_progress + self.resumable.chunksize() - 1,1046 self.resumable.size() - 1,1047 )1048 else:1049 data = self.resumable.getbytes(1050 self.resumable_progress, self.resumable.chunksize()1051 )1052 1053 # A short read implies that we are at EOF, so finish the upload.1054 if len(data) < self.resumable.chunksize():1055 size = str(self.resumable_progress + len(data))1056 1057 chunk_end = self.resumable_progress + len(data) - 11058 1059 headers = {1060 # Must set the content-length header here because httplib can't1061 # calculate the size when working with _StreamSlice.1062 "Content-Length": str(chunk_end - self.resumable_progress + 1),1063 }1064 1065 # An empty file results in chunk_end = -1 and size = 01066 # sending "bytes 0--1/0" results in an invalid request1067 # Only add header "Content-Range" if chunk_end != -11068 if chunk_end != -1:1069 headers["Content-Range"] = "bytes %d-%d/%s" % (1070 self.resumable_progress,1071 chunk_end,1072 size,1073 )1074 1075 for retry_num in range(num_retries + 1):1076 if retry_num > 0:1077 self._sleep(self._rand() * 2**retry_num)1078 LOGGER.warning(1079 "Retry #%d for media upload: %s %s, following status: %d"1080 % (retry_num, self.method, self.uri, resp.status)1081 )1082 1083 try:1084 resp, content = http.request(1085 self.resumable_uri, method="PUT", body=data, headers=headers1086 )1087 except:1088 self._in_error_state = True1089 raise1090 if not _should_retry_response(resp.status, content):1091 break1092 1093 return self._process_response(resp, content)1094 1095 def _process_response(self, resp, content):1096 """Process the response from a single chunk upload.1097 1098 Args:1099 resp: httplib2.Response, the response object.1100 content: string, the content of the response.1101 1102 Returns:1103 (status, body): (ResumableMediaStatus, object)1104 The body will be None until the resumable media is fully uploaded.1105 1106 Raises:1107 googleapiclient.errors.HttpError if the response was not a 2xx or a 308.1108 """1109 if resp.status in [200, 201]:1110 self._in_error_state = False1111 return None, self.postproc(resp, content)1112 elif resp.status == 308:1113 self._in_error_state = False1114 # A "308 Resume Incomplete" indicates we are not done.1115 try:1116 self.resumable_progress = int(resp["range"].split("-")[1]) + 11117 except KeyError:1118 # If resp doesn't contain range header, resumable progress is 01119 self.resumable_progress = 01120 if "location" in resp:1121 self.resumable_uri = resp["location"]1122 else:1123 self._in_error_state = True1124 raise HttpError(resp, content, uri=self.uri)1125 1126 return (1127 MediaUploadProgress(self.resumable_progress, self.resumable.size()),1128 None,1129 )1130 1131 def to_json(self):1132 """Returns a JSON representation of the HttpRequest."""1133 d = copy.copy(self.__dict__)1134 if d["resumable"] is not None:1135 d["resumable"] = self.resumable.to_json()1136 del d["http"]1137 del d["postproc"]1138 del d["_sleep"]1139 del d["_rand"]1140 1141 return json.dumps(d)1142 1143 @staticmethod1144 def from_json(s, http, postproc):1145 """Returns an HttpRequest populated with info from a JSON object."""1146 d = json.loads(s)1147 if d["resumable"] is not None:1148 d["resumable"] = MediaUpload.new_from_json(d["resumable"])1149 return HttpRequest(1150 http,1151 postproc,1152 uri=d["uri"],1153 method=d["method"],1154 body=d["body"],1155 headers=d["headers"],1156 methodId=d["methodId"],1157 resumable=d["resumable"],1158 )1159 1160 @staticmethod1161 def null_postproc(resp, contents):1162 return resp, contents1163 1164 1165class BatchHttpRequest(object):1166 """Batches multiple HttpRequest objects into a single HTTP request.1167 1168 Example:1169 from googleapiclient.http import BatchHttpRequest1170 1171 def list_animals(request_id, response, exception):1172 \"\"\"Do something with the animals list response.\"\"\"1173 if exception is not None:1174 # Do something with the exception.1175 pass1176 else:1177 # Do something with the response.1178 pass1179 1180 def list_farmers(request_id, response, exception):1181 \"\"\"Do something with the farmers list response.\"\"\"1182 if exception is not None:1183 # Do something with the exception.1184 pass1185 else:1186 # Do something with the response.1187 pass1188 1189 service = build('farm', 'v2')1190 1191 batch = BatchHttpRequest()1192 1193 batch.add(service.animals().list(), list_animals)1194 batch.add(service.farmers().list(), list_farmers)1195 batch.execute(http=http)1196 """1197 1198 @util.positional(1)1199 def __init__(self, callback=None, batch_uri=None):1200 """Constructor for a BatchHttpRequest.