codekingpro/portable-devtools

1# Copyright 2009 Google Inc.2#3# Licensed under the Apache License, Version 2.0 (the 'License')4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#         http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an 'AS IS' BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15"""Python implementation of the UA parser."""16 17from __future__ import absolute_import18 19import os20import re21import sys22import warnings23 24__author__ = "Lindsey Simon <elsigh@gmail.com>"25 26 27class UserAgentParser(object):28    def __init__(29        self, pattern, family_replacement=None, v1_replacement=None, v2_replacement=None30    ):31        """Initialize UserAgentParser.32 33        Args:34          pattern: a regular expression string35          family_replacement: a string to override the matched family (optional)36          v1_replacement: a string to override the matched v1 (optional)37          v2_replacement: a string to override the matched v2 (optional)38        """39        self.pattern = pattern40        self.user_agent_re = re.compile(self.pattern)41        self.family_replacement = family_replacement42        self.v1_replacement = v1_replacement43        self.v2_replacement = v2_replacement44 45    def MatchSpans(self, user_agent_string):46        match_spans = []47        match = self.user_agent_re.search(user_agent_string)48        if match:49            match_spans = [50                match.span(group_index) for group_index in range(1, match.lastindex + 1)51            ]52        return match_spans53 54    def Parse(self, user_agent_string):55        family, v1, v2, v3 = None, None, None, None56        match = self.user_agent_re.search(user_agent_string)57        if match:58            if self.family_replacement:59                if re.search(r"\$1", self.family_replacement):60                    family = re.sub(r"\$1", match.group(1), self.family_replacement)61                else:62                    family = self.family_replacement63            else:64                family = match.group(1)65 66            if self.v1_replacement:67                v1 = self.v1_replacement68            elif match.lastindex and match.lastindex >= 2:69                v1 = match.group(2) or None70 71            if self.v2_replacement:72                v2 = self.v2_replacement73            elif match.lastindex and match.lastindex >= 3:74                v2 = match.group(3) or None75 76            if match.lastindex and match.lastindex >= 4:77                v3 = match.group(4) or None78 79        return family, v1, v2, v380 81 82class OSParser(object):83    def __init__(84        self,85        pattern,86        os_replacement=None,87        os_v1_replacement=None,88        os_v2_replacement=None,89        os_v3_replacement=None,90        os_v4_replacement=None,91    ):92        """Initialize UserAgentParser.93 94        Args:95          pattern: a regular expression string96          os_replacement: a string to override the matched os (optional)97          os_v1_replacement: a string to override the matched v1 (optional)98          os_v2_replacement: a string to override the matched v2 (optional)99          os_v3_replacement: a string to override the matched v3 (optional)100          os_v4_replacement: a string to override the matched v4 (optional)101        """102        self.pattern = pattern103        self.user_agent_re = re.compile(self.pattern)104        self.os_replacement = os_replacement105        self.os_v1_replacement = os_v1_replacement106        self.os_v2_replacement = os_v2_replacement107        self.os_v3_replacement = os_v3_replacement108        self.os_v4_replacement = os_v4_replacement109 110    def MatchSpans(self, user_agent_string):111        match_spans = []112        match = self.user_agent_re.search(user_agent_string)113        if match:114            match_spans = [115                match.span(group_index) for group_index in range(1, match.lastindex + 1)116            ]117        return match_spans118 119    def Parse(self, user_agent_string):120        os, os_v1, os_v2, os_v3, os_v4 = None, None, None, None, None121        match = self.user_agent_re.search(user_agent_string)122        if match:123            if self.os_replacement:124                os = MultiReplace(self.os_replacement, match)125            elif match.lastindex:126                os = match.group(1)127 128            if self.os_v1_replacement:129                os_v1 = MultiReplace(self.os_v1_replacement, match)130            elif match.lastindex and match.lastindex >= 2:131                os_v1 = match.group(2)132 133            if self.os_v2_replacement:134                os_v2 = MultiReplace(self.os_v2_replacement, match)135            elif match.lastindex and match.lastindex >= 3:136                os_v2 = match.group(3)137 138            if self.os_v3_replacement:139                os_v3 = MultiReplace(self.os_v3_replacement, match)140            elif match.lastindex and match.lastindex >= 4:141                os_v3 = match.group(4)142 143            if self.os_v4_replacement:144                os_v4 = MultiReplace(self.os_v4_replacement, match)145            elif match.lastindex and match.lastindex >= 5:146                os_v4 = match.group(5)147 148        return os, os_v1, os_v2, os_v3, os_v4149 150 151def MultiReplace(string, match):152    def _repl(m):153        index = int(m.group(1)) - 1154        group = match.groups()155        if index < len(group):156            return group[index]157        return ""158 159    _string = re.sub(r"\$(\d)", _repl, string)160    _string = re.sub(r"^\s+|\s+$", "", _string)161    if _string == "":162        return None163    return _string164 165 166class DeviceParser(object):167    def __init__(168        self,169        pattern,170        regex_flag=None,171        device_replacement=None,172        brand_replacement=None,173        model_replacement=None,174    ):175        """Initialize UserAgentParser.176 177        Args:178          pattern: a regular expression string179          device_replacement: a string to override the matched device (optional)180        """181        self.pattern = pattern182        if regex_flag == "i":183            self.user_agent_re = re.compile(self.pattern, re.IGNORECASE)184        else:185            self.user_agent_re = re.compile(self.pattern)186        self.device_replacement = device_replacement187        self.brand_replacement = brand_replacement188        self.model_replacement = model_replacement189 190    def MatchSpans(self, user_agent_string):191        match_spans = []192        match = self.user_agent_re.search(user_agent_string)193        if match:194            match_spans = [195                match.span(group_index) for group_index in range(1, match.lastindex + 1)196            ]197        return match_spans198 199    def Parse(self, user_agent_string):200        device, brand, model = None, None, None201        match = self.user_agent_re.search(user_agent_string)202        if match:203            if self.device_replacement:204                device = MultiReplace(self.device_replacement, match)205            else:206                device = match.group(1)207 208            if self.brand_replacement:209                brand = MultiReplace(self.brand_replacement, match)210 211            if self.model_replacement:212                model = MultiReplace(self.model_replacement, match)213            elif len(match.groups()) > 0:214                model = match.group(1)215 216        return device, brand, model217 218 219MAX_CACHE_SIZE = 200220_PARSE_CACHE = {}221 222_UA_TYPES = str223if sys.version_info < (3,):224    _UA_TYPES = (str, unicode)225 226 227def _lookup(ua, args):228    if not isinstance(ua, _UA_TYPES):229        raise TypeError("Expected user agent to be a string, got %r" % ua)230 231    key = (ua, tuple(sorted(args.items())))232    entry = _PARSE_CACHE.get(key)233    if entry is not None:234        return entry235 236    if len(_PARSE_CACHE) >= MAX_CACHE_SIZE:237        _PARSE_CACHE.clear()238 239    v = _PARSE_CACHE[key] = {"string": ua}240    return v241 242 243def _cached(ua, args, key, fn):244    entry = _lookup(ua, args)245    r = entry.get(key)246    if not r:247        r = entry[key] = fn(ua, args)248    return r249 250 251def Parse(user_agent_string, **jsParseBits):252    """Parse all the things253    Args:254      user_agent_string: the full user agent string255    Returns:256      A dictionary containing all parsed bits257    """258    entry = _lookup(user_agent_string, jsParseBits)259    # entry is complete, return directly260    if len(entry) == 4:261        return entry262 263    # entry is partially or entirely empty264    if "user_agent" not in entry:265        entry["user_agent"] = _ParseUserAgent(user_agent_string, jsParseBits)266    if "os" not in entry:267        entry["os"] = _ParseOS(user_agent_string, jsParseBits)268    if "device" not in entry:269        entry["device"] = _ParseDevice(user_agent_string, jsParseBits)270 271    return entry272 273 274def ParseUserAgent(user_agent_string, **jsParseBits):275    """Parses the user-agent string for user agent (browser) info.276    Args:277      user_agent_string: The full user-agent string.278    Returns:279      A dictionary containing parsed bits.280    """281    return _cached(user_agent_string, jsParseBits, "user_agent", _ParseUserAgent)282 283 284def _ParseUserAgent(user_agent_string, jsParseBits):285    if jsParseBits:286        warnings.warn(287            "javascript overrides are deprecated and will be removed next release",288            category=DeprecationWarning,289            stacklevel=2,290        )291    if (292        "js_user_agent_family" in jsParseBits293        and jsParseBits["js_user_agent_family"] != ""294    ):295        family = jsParseBits["js_user_agent_family"]296        v1 = jsParseBits.get("js_user_agent_v1") or None297        v2 = jsParseBits.get("js_user_agent_v2") or None298        v3 = jsParseBits.get("js_user_agent_v3") or None299    else:300        for uaParser in USER_AGENT_PARSERS:301            family, v1, v2, v3 = uaParser.Parse(user_agent_string)302            if family:303                break304 305    # Override for Chrome Frame IFF Chrome is enabled.306    if "js_user_agent_string" in jsParseBits:307        js_user_agent_string = jsParseBits["js_user_agent_string"]308        if (309            js_user_agent_string310            and js_user_agent_string.find("Chrome/") > -1311            and user_agent_string.find("chromeframe") > -1312        ):313            jsOverride = {}314            jsOverride = ParseUserAgent(js_user_agent_string)315            family = "Chrome Frame (%s %s)" % (family, v1)316            v1 = jsOverride["major"]317            v2 = jsOverride["minor"]318            v3 = jsOverride["patch"]319 320    family = family or "Other"321    return {322        "family": family,323        "major": v1 or None,324        "minor": v2 or None,325        "patch": v3 or None,326    }327 328 329def ParseOS(user_agent_string, **jsParseBits):330    """Parses the user-agent string for operating system info331    Args:332      user_agent_string: The full user-agent string.333    Returns:334      A dictionary containing parsed bits.335    """336    return _cached(user_agent_string, jsParseBits, "os", _ParseOS)337 338 339def _ParseOS(user_agent_string, jsParseBits):340    if jsParseBits:341        warnings.warn(342            "javascript overrides are deprecated and will be removed next release",343            category=DeprecationWarning,344            stacklevel=2,345        )346    for osParser in OS_PARSERS:347        os, os_v1, os_v2, os_v3, os_v4 = osParser.Parse(user_agent_string)348        if os:349            break350    os = os or "Other"351    return {352        "family": os,353        "major": os_v1,354        "minor": os_v2,355        "patch": os_v3,356        "patch_minor": os_v4,357    }358 359 360def ParseDevice(user_agent_string, **jsParseBits):361    """Parses the user-agent string for device info.362    Args:363        user_agent_string: The full user-agent string.364    Returns:365        A dictionary containing parsed bits.366    """367    return _cached(user_agent_string, jsParseBits, "device", _ParseDevice)368 369 370def _ParseDevice(user_agent_string, jsParseBits):371    if jsParseBits:372        warnings.warn(373            "javascript overrides are deprecated and will be removed next release",374            category=DeprecationWarning,375            stacklevel=2,376        )377    for deviceParser in DEVICE_PARSERS:378        device, brand, model = deviceParser.Parse(user_agent_string)379        if device:380            break381 382    if device is None:383        device = "Other"384 385    return {"family": device, "brand": brand, "model": model}386 387 388def PrettyUserAgent(family, v1=None, v2=None, v3=None):389    """Pretty user agent string."""390    if v3:391        if v3[0].isdigit():392            return "%s %s.%s.%s" % (family, v1, v2, v3)393        else:394            return "%s %s.%s%s" % (family, v1, v2, v3)395    elif v2:396        return "%s %s.%s" % (family, v1, v2)397    elif v1:398        return "%s %s" % (family, v1)399    return family400 401 402def PrettyOS(os, os_v1=None, os_v2=None, os_v3=None, os_v4=None):403    """Pretty os string."""404    if os_v4:405        return "%s %s.%s.%s.%s" % (os, os_v1, os_v2, os_v3, os_v4)406    if os_v3:407        if os_v3[0].isdigit():408            return "%s %s.%s.%s" % (os, os_v1, os_v2, os_v3)409        else:410            return "%s %s.%s%s" % (os, os_v1, os_v2, os_v3)411    elif os_v2:412        return "%s %s.%s" % (os, os_v1, os_v2)413    elif os_v1:414        return "%s %s" % (os, os_v1)415    return os416 417 418def ParseWithJSOverrides(419    user_agent_string,420    js_user_agent_string=None,421    js_user_agent_family=None,422    js_user_agent_v1=None,423    js_user_agent_v2=None,424    js_user_agent_v3=None,425):426    """backwards compatible. use one of the other Parse methods instead!"""427    warnings.warn(428        "Use Parse (or a specialised parser)", DeprecationWarning, stacklevel=2429    )430 431    # Override via JS properties.432    if js_user_agent_family is not None and js_user_agent_family != "":433        family = js_user_agent_family434        v1 = None435        v2 = None436        v3 = None437        if js_user_agent_v1 is not None:438            v1 = js_user_agent_v1439        if js_user_agent_v2 is not None:440            v2 = js_user_agent_v2441        if js_user_agent_v3 is not None:442            v3 = js_user_agent_v3443    else:444        for parser in USER_AGENT_PARSERS:445            family, v1, v2, v3 = parser.Parse(user_agent_string)446            if family:447                break448 449    # Override for Chrome Frame IFF Chrome is enabled.450    if (451        js_user_agent_string452        and js_user_agent_string.find("Chrome/") > -1453        and user_agent_string.find("chromeframe") > -1454    ):455        family = "Chrome Frame (%s %s)" % (family, v1)456        ua_dict = ParseUserAgent(js_user_agent_string)457        v1 = ua_dict["major"]458        v2 = ua_dict["minor"]459        v3 = ua_dict["patch"]460 461    return family or "Other", v1, v2, v3462 463 464def Pretty(family, v1=None, v2=None, v3=None):465    """backwards compatible. use PrettyUserAgent instead!"""466    warnings.warn("Use PrettyUserAgent", DeprecationWarning, stacklevel=2)467    if v3:468        if v3[0].isdigit():469            return "%s %s.%s.%s" % (family, v1, v2, v3)470        else:471            return "%s %s.%s%s" % (family, v1, v2, v3)472    elif v2:473        return "%s %s.%s" % (family, v1, v2)474    elif v1:475        return "%s %s" % (family, v1)476    return family477 478 479def GetFilters(480    user_agent_string,481    js_user_agent_string=None,482    js_user_agent_family=None,483    js_user_agent_v1=None,484    js_user_agent_v2=None,485    js_user_agent_v3=None,486):487    """Return the optional arguments that should be saved and used to query.488 489    js_user_agent_string is always returned if it is present. We really only need490    it for Chrome Frame. However, I added it in the generally case to find other491    cases when it is different. When the recording of js_user_agent_string was492    added, we created new records for all new user agents.493 494    Since we only added js_document_mode for the IE 9 preview case, it did not495    cause new user agent records the way js_user_agent_string did.496 497    js_document_mode has since been removed in favor of individual property498    overrides.499 500    Args:501      user_agent_string: The full user-agent string.502      js_user_agent_string: JavaScript ua string from client-side503      js_user_agent_family: This is an override for the family name to deal504          with the fact that IE platform preview (for instance) cannot be505          distinguished by user_agent_string, but only in javascript.506      js_user_agent_v1: v1 override - see above.507      js_user_agent_v2: v1 override - see above.508      js_user_agent_v3: v1 override - see above.509    Returns:510      {js_user_agent_string: '[...]', js_family_name: '[...]', etc...}511    """512    filters = {}513    filterdict = {514        "js_user_agent_string": js_user_agent_string,515        "js_user_agent_family": js_user_agent_family,516        "js_user_agent_v1": js_user_agent_v1,517        "js_user_agent_v2": js_user_agent_v2,518        "js_user_agent_v3": js_user_agent_v3,519    }520    for key, value in filterdict.items():521        if value is not None and value != "":522            filters[key] = value523    return filters524 525 526# Build the list of user agent parsers from YAML527UA_PARSER_YAML = os.environ.get("UA_PARSER_YAML")528if UA_PARSER_YAML:529    # This will raise an ImportError if missing, obviously since it's no530    # longer a requirement531    import yaml532 533    try:534        # Try and use libyaml bindings if available since faster,535        # pyyaml doesn't do it by default (yaml/pyyaml#436)536        from yaml import CSafeLoader as SafeLoader537    except ImportError:538        from yaml import SafeLoader539 540    with open(UA_PARSER_YAML, "rb") as fp:541        regexes = yaml.load(fp, Loader=SafeLoader)542 543    USER_AGENT_PARSERS = []544    for _ua_parser in regexes["user_agent_parsers"]:545        _regex = _ua_parser["regex"]546 547        _family_replacement = _ua_parser.get("family_replacement")548        _v1_replacement = _ua_parser.get("v1_replacement")549        _v2_replacement = _ua_parser.get("v2_replacement")550 551        USER_AGENT_PARSERS.append(552            UserAgentParser(553                _regex, _family_replacement, _v1_replacement, _v2_replacement554            )555        )556 557    OS_PARSERS = []558    for _os_parser in regexes["os_parsers"]:559        _regex = _os_parser["regex"]560 561        _os_replacement = _os_parser.get("os_replacement")562        _os_v1_replacement = _os_parser.get("os_v1_replacement")563        _os_v2_replacement = _os_parser.get("os_v2_replacement")564        _os_v3_replacement = _os_parser.get("os_v3_replacement")565        _os_v4_replacement = _os_parser.get("os_v4_replacement")566 567        OS_PARSERS.append(568            OSParser(569                _regex,570                _os_replacement,571                _os_v1_replacement,572                _os_v2_replacement,573                _os_v3_replacement,574                _os_v4_replacement,575            )576        )577 578    DEVICE_PARSERS = []579    for _device_parser in regexes["device_parsers"]:580        _regex = _device_parser["regex"]581 582        _regex_flag = _device_parser.get("regex_flag")583        _device_replacement = _device_parser.get("device_replacement")584        _brand_replacement = _device_parser.get("brand_replacement")585        _model_replacement = _device_parser.get("model_replacement")586 587        DEVICE_PARSERS.append(588            DeviceParser(589                _regex,590                _regex_flag,591                _device_replacement,592                _brand_replacement,593                _model_replacement,594            )595        )596 597    # Clean our our temporary vars explicitly598    # so they can't be reused or imported599    del regexes600    del yaml601    del SafeLoader602else:603    # Just load our pre-compiled versions604    from ._regexes import USER_AGENT_PARSERS, DEVICE_PARSERS, OS_PARSERS605