codekingpro/portable-devtools
114k
1# Copyright 2009 Facebook
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""Translation methods for generating localized strings.
16
17To load a locale and generate a translated string::
18
19 user_locale = tornado.locale.get("es_LA")
20 print(user_locale.translate("Sign out"))
21
22`tornado.locale.get()` returns the closest matching locale, not necessarily the
23specific locale you requested. You can support pluralization with
24additional arguments to `~Locale.translate()`, e.g.::
25
26 people = [...]
27 message = user_locale.translate(
28 "%(list)s is online", "%(list)s are online", len(people))
29 print(message % {"list": user_locale.list(people)})
30
31The first string is chosen if ``len(people) == 1``, otherwise the second
32string is chosen.
33
34Applications should call one of `load_translations` (which uses a simple
35CSV format) or `load_gettext_translations` (which uses the ``.mo`` format
36supported by `gettext` and related tools). If neither method is called,
37the `Locale.translate` method will simply return the original string.
38"""
39
40import codecs
41import csv
42import datetime
43import gettext
44import glob
45import os
46import re
47
48from tornado import escape
49from tornado.log import gen_log
50
51from tornado._locale_data import LOCALE_NAMES
52
53from typing import Iterable, Any, Union, Dict, Optional
54
55_default_locale = "en_US"
56_translations = {} # type: Dict[str, Any]
57_supported_locales = frozenset([_default_locale])
58_use_gettext = False
59CONTEXT_SEPARATOR = "\x04"
60
61
62def get(*locale_codes: str) -> "Locale":
63 """Returns the closest match for the given locale codes.
64
65 We iterate over all given locale codes in order. If we have a tight
66 or a loose match for the code (e.g., "en" for "en_US"), we return
67 the locale. Otherwise we move to the next code in the list.
68
69 By default we return ``en_US`` if no translations are found for any of
70 the specified locales. You can change the default locale with
71 `set_default_locale()`.
72 """
73 return Locale.get_closest(*locale_codes)
74
75
76def set_default_locale(code: str) -> None:
77 """Sets the default locale.
78
79 The default locale is assumed to be the language used for all strings
80 in the system. The translations loaded from disk are mappings from
81 the default locale to the destination locale. Consequently, you don't
82 need to create a translation file for the default locale.
83 """
84 global _default_locale
85 global _supported_locales
86 _default_locale = code
87 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])
88
89
90def load_translations(directory: str, encoding: Optional[str] = None) -> None:
91 """Loads translations from CSV files in a directory.
92
93 Translations are strings with optional Python-style named placeholders
94 (e.g., ``My name is %(name)s``) and their associated translations.
95
96 The directory should have translation files of the form ``LOCALE.csv``,
97 e.g. ``es_GT.csv``. The CSV files should have two or three columns: string,
98 translation, and an optional plural indicator. Plural indicators should
99 be one of "plural" or "singular". A given string can have both singular
100 and plural forms. For example ``%(name)s liked this`` may have a
101 different verb conjugation depending on whether %(name)s is one
102 name or a list of names. There should be two rows in the CSV file for
103 that string, one with plural indicator "singular", and one "plural".
104 For strings with no verbs that would change on translation, simply
105 use "unknown" or the empty string (or don't include the column at all).
106
107 The file is read using the `csv` module in the default "excel" dialect.
108 In this format there should not be spaces after the commas.
109
110 If no ``encoding`` parameter is given, the encoding will be
111 detected automatically (among UTF-8 and UTF-16) if the file
112 contains a byte-order marker (BOM), defaulting to UTF-8 if no BOM
113 is present.
114
115 Example translation ``es_LA.csv``::
116
117 "I love you","Te amo"
118 "%(name)s liked this","A %(name)s les gustó esto","plural"
119 "%(name)s liked this","A %(name)s le gustó esto","singular"
120
121 .. versionchanged:: 4.3
122 Added ``encoding`` parameter. Added support for BOM-based encoding
123 detection, UTF-16, and UTF-8-with-BOM.
124 """
125 global _translations
126 global _supported_locales
127 _translations = {}
128 for path in os.listdir(directory):
129 if not path.endswith(".csv"):
130 continue
131 locale, extension = path.split(".")
132 if not re.match("[a-z]+(_[A-Z]+)?$", locale):
133 gen_log.error(
134 "Unrecognized locale %r (path: %s)",
135 locale,
136 os.path.join(directory, path),
137 )
138 continue
139 full_path = os.path.join(directory, path)
140 if encoding is None:
141 # Try to autodetect encoding based on the BOM.
142 with open(full_path, "rb") as bf:
143 data = bf.read(len(codecs.BOM_UTF16_LE))
144 if data in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
145 encoding = "utf-16"
146 else:
147 # utf-8-sig is "utf-8 with optional BOM". It's discouraged
148 # in most cases but is common with CSV files because Excel
149 # cannot read utf-8 files without a BOM.
150 encoding = "utf-8-sig"
151 # python 3: csv.reader requires a file open in text mode.
152 # Specify an encoding to avoid dependence on $LANG environment variable.
153 with open(full_path, encoding=encoding) as f:
154 _translations[locale] = {}
155 for i, row in enumerate(csv.reader(f)):
156 if not row or len(row) < 2:
157 continue
158 row = [escape.to_unicode(c).strip() for c in row]
159 english, translation = row[:2]
160 if len(row) > 2:
161 plural = row[2] or "unknown"
162 else:
163 plural = "unknown"
164 if plural not in ("plural", "singular", "unknown"):
165 gen_log.error(
166 "Unrecognized plural indicator %r in %s line %d",
167 plural,
168 path,
169 i + 1,
170 )
171 continue
172 _translations[locale].setdefault(plural, {})[english] = translation
173 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])
174 gen_log.debug("Supported locales: %s", sorted(_supported_locales))
175
176
177def load_gettext_translations(directory: str, domain: str) -> None:
178 """Loads translations from `gettext`'s locale tree
179
180 Locale tree is similar to system's ``/usr/share/locale``, like::
181
182 {directory}/{lang}/LC_MESSAGES/{domain}.mo
183
184 Three steps are required to have your app translated:
185
186 1. Generate POT translation file::
187
188 xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc
189
190 2. Merge against existing POT file::
191
192 msgmerge old.po mydomain.po > new.po
193
194 3. Compile::
195
196 msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo
197 """
198 global _translations
199 global _supported_locales
200 global _use_gettext
201 _translations = {}
202
203 for filename in glob.glob(
204 os.path.join(directory, "*", "LC_MESSAGES", domain + ".mo")
205 ):
206 lang = os.path.basename(os.path.dirname(os.path.dirname(filename)))
207 try:
208 _translations[lang] = gettext.translation(
209 domain, directory, languages=[lang]
210 )
211 except Exception as e:
212 gen_log.error("Cannot load translation for '%s': %s", lang, str(e))
213 continue
214 _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])
215 _use_gettext = True
216 gen_log.debug("Supported locales: %s", sorted(_supported_locales))
217
218
219def get_supported_locales() -> Iterable[str]:
220 """Returns a list of all the supported locale codes."""
221 return _supported_locales
222
223
224class Locale:
225 """Object representing a locale.
226
227 After calling one of `load_translations` or `load_gettext_translations`,
228 call `get` or `get_closest` to get a Locale object.
229 """
230
231 _cache = {} # type: Dict[str, Locale]
232
233 @classmethod
234 def get_closest(cls, *locale_codes: str) -> "Locale":
235 """Returns the closest match for the given locale code."""
236 for code in locale_codes:
237 if not code:
238 continue
239 code = code.replace("-", "_")
240 parts = code.split("_")
241 if len(parts) > 2:
242 continue
243 elif len(parts) == 2:
244 code = parts[0].lower() + "_" + parts[1].upper()
245 if code in _supported_locales:
246 return cls.get(code)
247 if parts[0].lower() in _supported_locales:
248 return cls.get(parts[0].lower())
249 return cls.get(_default_locale)
250
251 @classmethod
252 def get(cls, code: str) -> "Locale":
253 """Returns the Locale for the given locale code.
254
255 If it is not supported, we raise an exception.
256 """
257 if code not in cls._cache:
258 assert code in _supported_locales
259 translations = _translations.get(code, None)
260 if translations is None:
261 locale = CSVLocale(code, {}) # type: Locale
262 elif _use_gettext:
263 locale = GettextLocale(code, translations)
264 else:
265 locale = CSVLocale(code, translations)
266 cls._cache[code] = locale
267 return cls._cache[code]
268
269 def __init__(self, code: str) -> None:
270 self.code = code
271 self.name = LOCALE_NAMES.get(code, {}).get("name", "Unknown")
272 self.rtl = False
273 for prefix in ["fa", "ar", "he"]:
274 if self.code.startswith(prefix):
275 self.rtl = True
276 break
277
278 # Initialize strings for date formatting
279 _ = self.translate
280 self._months = [
281 _("January"),
282 _("February"),
283 _("March"),
284 _("April"),
285 _("May"),
286 _("June"),
287 _("July"),
288 _("August"),
289 _("September"),
290 _("October"),
291 _("November"),
292 _("December"),
293 ]
294 self._weekdays = [
295 _("Monday"),
296 _("Tuesday"),
297 _("Wednesday"),
298 _("Thursday"),
299 _("Friday"),
300 _("Saturday"),
301 _("Sunday"),
302 ]
303
304 def translate(
305 self,
306 message: str,
307 plural_message: Optional[str] = None,
308 count: Optional[int] = None,
309 ) -> str:
310 """Returns the translation for the given message for this locale.
311
312 If ``plural_message`` is given, you must also provide
313 ``count``. We return ``plural_message`` when ``count != 1``,
314 and we return the singular form for the given message when
315 ``count == 1``.
316 """
317 raise NotImplementedError()
318
319 def pgettext(
320 self,
321 context: str,
322 message: str,
323 plural_message: Optional[str] = None,
324 count: Optional[int] = None,
325 ) -> str:
326 raise NotImplementedError()
327
328 def format_date(
329 self,
330 date: Union[int, float, datetime.datetime],
331 gmt_offset: int = 0,
332 relative: bool = True,
333 shorter: bool = False,
334 full_format: bool = False,
335 ) -> str:
336 """Formats the given date.
337
338 By default, we return a relative time (e.g., "2 minutes ago"). You
339 can return an absolute date string with ``relative=False``.
340
341 You can force a full format date ("July 10, 1980") with
342 ``full_format=True``.
343
344 This method is primarily intended for dates in the past.
345 For dates in the future, we fall back to full format.
346
347 .. versionchanged:: 6.4
348 Aware `datetime.datetime` objects are now supported (naive
349 datetimes are still assumed to be UTC).
350 """
351 if isinstance(date, (int, float)):
352 date = datetime.datetime.fromtimestamp(date, datetime.timezone.utc)
353 if date.tzinfo is None:
354 date = date.replace(tzinfo=datetime.timezone.utc)
355 now = datetime.datetime.now(datetime.timezone.utc)
356 if date > now:
357 if relative and (date - now).seconds < 60:
358 # Due to click skew, things are some things slightly
359 # in the future. Round timestamps in the immediate
360 # future down to now in relative mode.
361 date = now
362 else:
363 # Otherwise, future dates always use the full format.
364 full_format = True
365 local_date = date - datetime.timedelta(minutes=gmt_offset)
366 local_now = now - datetime.timedelta(minutes=gmt_offset)
367 local_yesterday = local_now - datetime.timedelta(hours=24)
368 difference = now - date
369 seconds = difference.seconds
370 days = difference.days
371
372 _ = self.translate
373 format = None
374 if not full_format:
375 if relative and days == 0:
376 if seconds < 50:
377 return _("1 second ago", "%(seconds)d seconds ago", seconds) % {
378 "seconds": seconds
379 }
380
381 if seconds < 50 * 60:
382 minutes = round(seconds / 60.0)
383 return _("1 minute ago", "%(minutes)d minutes ago", minutes) % {
384 "minutes": minutes
385 }
386
387 hours = round(seconds / (60.0 * 60))
388 return _("1 hour ago", "%(hours)d hours ago", hours) % {"hours": hours}
389
390 if days == 0:
391 format = _("%(time)s")
392 elif days == 1 and local_date.day == local_yesterday.day and relative:
393 format = _("yesterday") if shorter else _("yesterday at %(time)s")
394 elif days < 5:
395 format = _("%(weekday)s") if shorter else _("%(weekday)s at %(time)s")
396 elif days < 334: # 11mo, since confusing for same month last year
397 format = (
398 _("%(month_name)s %(day)s")
399 if shorter
400 else _("%(month_name)s %(day)s at %(time)s")
401 )
402
403 if format is None:
404 format = (
405 _("%(month_name)s %(day)s, %(year)s")
406 if shorter
407 else _("%(month_name)s %(day)s, %(year)s at %(time)s")
408 )
409
410 tfhour_clock = self.code not in ("en", "en_US", "zh_CN")
411 if tfhour_clock:
412 str_time = "%d:%02d" % (local_date.hour, local_date.minute)
413 elif self.code == "zh_CN":
414 str_time = "%s%d:%02d" % (
415 ("\u4e0a\u5348", "\u4e0b\u5348")[local_date.hour >= 12],
416 local_date.hour % 12 or 12,
417 local_date.minute,
418 )
419 else:
420 str_time = "%d:%02d %s" % (
421 local_date.hour % 12 or 12,
422 local_date.minute,
423 ("am", "pm")[local_date.hour >= 12],
424 )
425
426 return format % {
427 "month_name": self._months[local_date.month - 1],
428 "weekday": self._weekdays[local_date.weekday()],
429 "day": str(local_date.day),
430 "year": str(local_date.year),
431 "time": str_time,
432 }
433
434 def format_day(
435 self, date: datetime.datetime, gmt_offset: int = 0, dow: bool = True
436 ) -> bool:
437 """Formats the given date as a day of week.
438
439 Example: "Monday, January 22". You can remove the day of week with
440 ``dow=False``.
441 """
442 local_date = date - datetime.timedelta(minutes=gmt_offset)
443 _ = self.translate
444 if dow:
445 return _("%(weekday)s, %(month_name)s %(day)s") % {
446 "month_name": self._months[local_date.month - 1],
447 "weekday": self._weekdays[local_date.weekday()],
448 "day": str(local_date.day),
449 }
450 else:
451 return _("%(month_name)s %(day)s") % {
452 "month_name": self._months[local_date.month - 1],
453 "day": str(local_date.day),
454 }
455
456 def list(self, parts: Any) -> str:
457 """Returns a comma-separated list for the given list of parts.
458
459 The format is, e.g., "A, B and C", "A and B" or just "A" for lists
460 of size 1.
461 """
462 _ = self.translate
463 if len(parts) == 0:
464 return ""
465 if len(parts) == 1:
466 return parts[0]
467 comma = " \u0648 " if self.code.startswith("fa") else ", "
468 return _("%(commas)s and %(last)s") % {
469 "commas": comma.join(parts[:-1]),
470 "last": parts[len(parts) - 1],
471 }
472
473 def friendly_number(self, value: int) -> str:
474 """Returns a comma-separated number for the given integer."""
475 if self.code not in ("en", "en_US"):
476 return str(value)
477 s = str(value)
478 parts = []
479 while s:
480 parts.append(s[-3:])
481 s = s[:-3]
482 return ",".join(reversed(parts))
483
484
485class CSVLocale(Locale):
486 """Locale implementation using tornado's CSV translation format."""
487
488 def __init__(self, code: str, translations: Dict[str, Dict[str, str]]) -> None:
489 self.translations = translations
490 super().__init__(code)
491
492 def translate(
493 self,
494 message: str,
495 plural_message: Optional[str] = None,
496 count: Optional[int] = None,
497 ) -> str:
498 if plural_message is not None:
499 assert count is not None
500 if count != 1:
501 message = plural_message
502 message_dict = self.translations.get("plural", {})
503 else:
504 message_dict = self.translations.get("singular", {})
505 else:
506 message_dict = self.translations.get("unknown", {})
507 return message_dict.get(message, message)
508
509 def pgettext(
510 self,
511 context: str,
512 message: str,
513 plural_message: Optional[str] = None,
514 count: Optional[int] = None,
515 ) -> str:
516 if self.translations:
517 gen_log.warning("pgettext is not supported by CSVLocale")
518 return self.translate(message, plural_message, count)
519
520
521class GettextLocale(Locale):
522 """Locale implementation using the `gettext` module."""
523
524 def __init__(self, code: str, translations: gettext.NullTranslations) -> None:
525 self.ngettext = translations.ngettext
526 self.gettext = translations.gettext
527 # self.gettext must exist before __init__ is called, since it
528 # calls into self.translate
529 super().__init__(code)
530
531 def translate(
532 self,
533 message: str,
534 plural_message: Optional[str] = None,
535 count: Optional[int] = None,
536 ) -> str:
537 if plural_message is not None:
538 assert count is not None
539 return self.ngettext(message, plural_message, count)
540 else:
541 return self.gettext(message)
542
543 def pgettext(
544 self,
545 context: str,
546 message: str,
547 plural_message: Optional[str] = None,
548 count: Optional[int] = None,
549 ) -> str:
550 """Allows to set context for translation, accepts plural forms.
551
552 Usage example::
553
554 pgettext("law", "right")
555 pgettext("good", "right")
556
557 Plural message example::
558
559 pgettext("organization", "club", "clubs", len(clubs))
560 pgettext("stick", "club", "clubs", len(clubs))
561
562 To generate POT file with context, add following options to step 1
563 of `load_gettext_translations` sequence::
564
565 xgettext [basic options] --keyword=pgettext:1c,2 --keyword=pgettext:1c,2,3
566
567 .. versionadded:: 4.2
568 """
569 if plural_message is not None:
570 assert count is not None
571 msgs_with_ctxt = (
572 f"{context}{CONTEXT_SEPARATOR}{message}",
573 f"{context}{CONTEXT_SEPARATOR}{plural_message}",
574 count,
575 )
576 result = self.ngettext(*msgs_with_ctxt)
577 if CONTEXT_SEPARATOR in result:
578 # Translation not found
579 result = self.ngettext(message, plural_message, count)
580 return result
581 else:
582 msg_with_ctxt = f"{context}{CONTEXT_SEPARATOR}{message}"
583 result = self.gettext(msg_with_ctxt)
584 if CONTEXT_SEPARATOR in result:
585 # Translation not found
586 result = message
587 return result
588