From 061c3cabb36934e77648af6fffcac6a62a6b3f1b Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Mon, 4 Feb 2019 16:18:53 +0100 Subject: [PATCH 01/19] Convert resolver to a pre-evaluated tree --- fluent.runtime/fluent/runtime/__init__.py | 16 +- fluent.runtime/fluent/runtime/prepare.py | 45 +++ fluent.runtime/fluent/runtime/resolver.py | 365 ++++++++++------------ 3 files changed, 228 insertions(+), 198 deletions(-) create mode 100644 fluent.runtime/fluent/runtime/prepare.py diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index befc1bb0..98f0cdd1 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -8,7 +8,8 @@ from fluent.syntax.ast import Message, Term from .builtins import BUILTINS -from .resolver import resolve +from .prepare import Compiler +from .resolver import ResolverEnvironment, CurrentEnvironment from .utils import ATTRIBUTE_SEPARATOR, TERM_SIGIL, add_message_and_attrs_to_store, ast_to_id @@ -33,6 +34,8 @@ def __init__(self, locales, functions=None, use_isolating=True): self._functions = _functions self._use_isolating = use_isolating self._messages_and_terms = {} + self._compiled = {} + self._compiler = Compiler() self._babel_locale = self._get_babel_locale() self._plural_form = babel.plural.to_python(self._babel_locale.plural_form) @@ -56,10 +59,17 @@ def has_message(self, message_id): def format(self, message_id, args=None): if message_id.startswith(TERM_SIGIL): raise LookupError(message_id) - message = self._messages_and_terms[message_id] if args is None: args = {} - return resolve(self, message, args) + if message_id not in self._compiled: + message = self._messages_and_terms[message_id] + self._compiled[message_id] = self._compiler(message.value) + resolve = self._compiled[message_id] + errors = [] + env = ResolverEnvironment(context=self, + current=CurrentEnvironment(args=args), + errors=errors) + return [resolve(env), errors] def _get_babel_locale(self): for l in self.locales: diff --git a/fluent.runtime/fluent/runtime/prepare.py b/fluent.runtime/fluent/runtime/prepare.py new file mode 100644 index 00000000..30139f9a --- /dev/null +++ b/fluent.runtime/fluent/runtime/prepare.py @@ -0,0 +1,45 @@ +from __future__ import absolute_import, unicode_literals +from fluent.syntax import ast as FTL +from . import resolver + + +class Compiler(object): + def __init__(self, bundle=None): + self.bundle = None + + def __call__(self, item): + if isinstance(item, FTL.BaseNode): + return self.compile(item) + if isinstance(item, (tuple, list)): + return [self(elem) for elem in item] + return item + + def compile(self, node): + nodename = type(node).__name__ + if not hasattr(resolver, nodename): + return node + kwargs = vars(node).copy() + for propname, propvalue in kwargs.items(): + kwargs[propname] = self(propvalue) + handler = getattr(self, 'compile_' + nodename, self.compile_generic) + return handler(nodename, **kwargs) + + def compile_generic(self, nodename, **kwargs): + return getattr(resolver, nodename)(**kwargs) + + def compile_Placeable(self, _, expression, **kwargs): + if isinstance(expression, resolver.Literal): + return expression + return resolver.Placeable(expression=expression, **kwargs) + + def compile_Pattern(self, _, elements, **kwargs): + if any( + not isinstance(child, resolver.Literal) + for child in elements + ): + return resolver.Pattern(elements=elements, **kwargs) + if len(elements) == 1: + return elements[0] + return resolver.TextElement( + ''.join(child(None) for child in elements) + ) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 07e03e40..919274a2 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -7,10 +7,7 @@ import attr import six -from fluent.syntax.ast import (Attribute, AttributeExpression, CallExpression, Identifier, Message, MessageReference, - NumberLiteral, Pattern, Placeable, SelectExpression, StringLiteral, Term, TermReference, - TextElement, VariableReference, VariantExpression, VariantList) - +from fluent.syntax import ast as FTL from .errors import FluentCyclicReferenceError, FluentFormatError, FluentReferenceError from .types import FluentDateType, FluentNone, FluentNumber, fluent_date, fluent_number from .utils import numeric_to_native, reference_to_id, unknown_reference_error_obj @@ -114,88 +111,76 @@ def handle(expr, env): .format(type(expr).__name__)) -@handle.register(Message) -def handle_message(message, env): - return handle(message.value, env) +class BaseResolver(object): + def __call__(self, env): + raise NotImplementedError + +class Literal(BaseResolver): + pass -@handle.register(Term) -def handle_term(term, env): - return handle(term.value, env) +class Message(FTL.Message, BaseResolver): + def __call__(self, env): + return self.value(env) -@handle.register(Pattern) -def handle_pattern(pattern, env): - if pattern in env.dirty: - env.errors.append(FluentCyclicReferenceError("Cyclic reference")) - return FluentNone() - env.dirty.add(pattern) +class Term(FTL.Term, BaseResolver): + def __call__(self, env): + return self.value(env) - parts = [] - use_isolating = env.context._use_isolating and len(pattern.elements) > 1 - for element in pattern.elements: - env.part_count += 1 - if env.part_count > MAX_PARTS: - if env.part_count == MAX_PARTS + 1: - # Only append an error once. - env.errors.append(ValueError("Too many parts in message (> {0}), " - "aborting.".format(MAX_PARTS))) - parts.append(fully_resolve(FluentNone(), env)) - break +class Pattern(FTL.Pattern, BaseResolver): + def __init__(self, *args, **kwargs): + super(Pattern, self).__init__(*args, **kwargs) + self.dirty = False + + def __call__(self, env): + if self.dirty: + env.errors.append(FluentCyclicReferenceError("Cyclic reference")) + return FluentNone() + self.dirty = True + retval = ''.join( + element(env) for element in self.elements + ) + self.dirty = False + return retval - if isinstance(element, TextElement): - # shortcut deliberately omits the FSI/PDI chars here. - parts.append(element.value) - continue - part = fully_resolve(element, env) - if use_isolating: - parts.append(FSI) - if len(part) > MAX_PART_LENGTH: - env.errors.append(ValueError( - "Too many characters in part, " - "({0}, max allowed is {1})".format(len(part), - MAX_PART_LENGTH))) - part = part[:MAX_PART_LENGTH] - parts.append(part) - if use_isolating: - parts.append(PDI) - retval = "".join(parts) - env.dirty.remove(pattern) - return retval +class TextElement(FTL.TextElement, Literal): + def __call__(self, env): + return self.value -@handle.register(TextElement) -def handle_text_element(text_element, env): - return text_element.value +class Placeable(FTL.Placeable, BaseResolver): + def __call__(self, env): + return self.expression(env) -@handle.register(Placeable) -def handle_placeable(placeable, env): - return handle(placeable.expression, env) +class StringLiteral(FTL.StringLiteral, Literal): + def __call__(self, env): + return self.value -@handle.register(StringLiteral) -def handle_string_expression(string_expression, env): - return string_expression.value +class NumberLiteral(FTL.NumberLiteral, Literal): + def __call__(self, env): + return self.value -@handle.register(NumberLiteral) -def handle_number_expression(number_expression, env): - return numeric_to_native(number_expression.value) +class MessageReference(FTL.MessageReference, BaseResolver): + def __call__(self, env): + return lookup_reference(self, env)(env) -@handle.register(MessageReference) -def handle_message_reference(message_reference, env): - return handle(lookup_reference(message_reference, env), env) +class TermReference(FTL.TermReference, BaseResolver): + def __call__(self, env): + with env.modified_for_term_reference(): + return lookup_reference(self, env)(env) -@handle.register(TermReference) -def handle_term_reference(term_reference, env): - with env.modified_for_term_reference(): - return handle(lookup_reference(term_reference, env), env) +class FluentNoneResolver(FluentNone, BaseResolver): + def __call__(self, env): + return self.format(env.context._babel_locale) def lookup_reference(ref, env): @@ -206,7 +191,7 @@ def lookup_reference(ref, env): ref_id = reference_to_id(ref) try: - return env.context._messages_and_terms[ref_id] + return env.context._compiler(env.context._messages_and_terms[ref_id]) except LookupError: env.errors.append(unknown_reference_error_obj(ref_id)) @@ -214,111 +199,98 @@ def lookup_reference(ref, env): # Fallback parent_id = reference_to_id(ref.ref) try: - return env.context._messages_and_terms[parent_id] + return env.context._compiler(env.context._messages_and_terms[parent_id]) except LookupError: # Don't add error here, because we already added error for the # actual thing we were looking for. pass - return FluentNone(ref_id) - - -@handle.register(FluentNone) -def handle_fluent_none(none, env): - return none.format(env.context._babel_locale) - - -@handle.register(type(None)) -def handle_none(none, env): - # We raise the same error type here as when a message is completely missing. - raise LookupError("Message body not defined") - - -@handle.register(VariableReference) -def handle_variable_reference(argument, env): - name = argument.id.name - try: - arg_val = env.current.args[name] - except LookupError: - if env.current.error_for_missing_arg: - env.errors.append( - FluentReferenceError("Unknown external: {0}".format(name))) - return FluentNone(name) - - if isinstance(arg_val, - (int, float, Decimal, - date, datetime, - text_type)): - return arg_val - env.errors.append(TypeError("Unsupported external type: {0}, {1}" - .format(name, type(arg_val)))) - return FluentNone(name) - - -@handle.register(AttributeExpression) -def handle_attribute_expression(attribute_ref, env): - return handle(lookup_reference(attribute_ref, env), env) - - -@handle.register(Attribute) -def handle_attribute(attribute, env): - return handle(attribute.value, env) - - -@handle.register(VariantList) -def handle_variant_list(variant_list, env): - return select_from_variant_list(variant_list, env, None) - - -def select_from_variant_list(variant_list, env, key): - found = None - for variant in variant_list.variants: - if variant.default: - default = variant - if key is None: - # We only want the default + return FluentNoneResolver(ref_id) + + +class VariableReference(FTL.VariableReference): + def __call__(self, env): + name = self.id.name + try: + arg_val = env.current.args[name] + except LookupError: + if env.current.error_for_missing_arg: + env.errors.append( + FluentReferenceError("Unknown external: {0}".format(name))) + return FluentNoneResolver(name) + + if isinstance(arg_val, + (int, float, Decimal, + date, datetime, + text_type)): + return lambda env: arg_val + env.errors.append(TypeError("Unsupported external type: {0}, {1}" + .format(name, type(arg_val)))) + return FluentNoneResolver(name) + + +class AttributeExpression(FTL.AttributeExpression, BaseResolver): + def __call__(self, env): + return lookup_reference(self, env)(env) + + +class Attribute(FTL.Attribute, BaseResolver): + def __call__(self, env): + return self.value(env) + + +class VariantListResolver(BaseResolver): + def select_from_variant_list(self, env, key): + found = None + for variant in self.variants: + if variant.default: + default = variant + if key is None: + # We only want the default + break + + compare_value = variant.key(env) + if match(key, compare_value, env): + found = variant break - compare_value = handle(variant.key, env) - if match(key, compare_value, env): - found = variant - break + if found is None: + if (key is not None and not isinstance(key, FluentNone)): + env.errors.append(FluentReferenceError("Unknown variant: {0}" + .format(key))) + found = default + if found is None: + return FluentNoneResolver() - if found is None: - if (key is not None and not isinstance(key, FluentNone)): - env.errors.append(FluentReferenceError("Unknown variant: {0}" - .format(key))) - found = default - if found is None: - return FluentNone() + return found.value(env) - return handle(found.value, env) +class VariantList(FTL.VariantList, VariantListResolver): + def __call__(self, env): + return self.select_from_variant_list(env, None) -@handle.register(SelectExpression) -def handle_select_expression(expression, env): - key = handle(expression.selector, env) - return select_from_select_expression(expression, env, - key=key) +class SelectExpression(FTL.SelectExpression, BaseResolver): + def __call__(self, env): + key = self.selector(env) + return self.select_from_select_expression(env, key=key) -def select_from_select_expression(expression, env, key): - default = None - found = None - for variant in expression.variants: - if variant.default: - default = variant + def select_from_select_expression(self, env, key): + default = None + found = None + for variant in self.variants: + if variant.default: + default = variant - compare_value = handle(variant.key, env) - if match(key, compare_value, env): - found = variant - break + if match(key, variant.key(env), env): + found = variant + break - if found is None: - found = default - if found is None: - return FluentNone() - return handle(found.value, env) + if found is None: + found = default + if found is None: + return FluentNoneResolver() + return found.value(env) def is_number(val): @@ -340,54 +312,57 @@ def match(val1, val2, env): return val1 == val2 -@handle.register(Identifier) -def handle_indentifier(identifier, env): - return identifier.name +class Variant(FTL.Variant, BaseResolver): + def __call__(self, env): + return self.value(env) -@handle.register(VariantExpression) -def handle_variant_expression(expression, env): - message = lookup_reference(expression.ref, env) - if isinstance(message, FluentNone): - return message +class Identifier(FTL.Identifier, BaseResolver): + def __call__(self, env): + return self.name - # TODO What to do if message is not a VariantList? - # Need test at least. - assert isinstance(message.value, VariantList) - variant_name = expression.key.name - return select_from_variant_list(message.value, - env, - variant_name) +class VariantExpression(FTL.VariantExpression, BaseResolver): + def __call__(self, env): + message = lookup_reference(self.ref, env) + if isinstance(message, FluentNone): + return FluentNoneResolver(FluentNone.name) + # TODO What to do if message is not a VariantList? + # Need test at least. + assert isinstance(message.value, VariantList) -@handle.register(CallExpression) -def handle_call_expression(expression, env): - args = [handle(arg, env) for arg in expression.positional] - kwargs = {kwarg.name.name: handle(kwarg.value, env) for kwarg in expression.named} + variant_name = self.key.name + return message.value.select_from_variant_list(env, variant_name) - if isinstance(expression.callee, (TermReference, AttributeExpression)): - term = lookup_reference(expression.callee, env) - if args: - env.errors.append(FluentFormatError("Ignored positional arguments passed to term '{0}'" - .format(reference_to_id(expression.callee)))) - with env.modified_for_term_reference(args=kwargs): - return handle(term, env) - # builtin or custom function call - function_name = expression.callee.id.name - try: - function = env.context._functions[function_name] - except LookupError: - env.errors.append(FluentReferenceError("Unknown function: {0}" - .format(function_name))) - return FluentNone(function_name + "()") +class CallExpression(FTL.CallExpression, BaseResolver): + def __call__(self, env): + args = [arg(env) for arg in self.positional] + kwargs = {kwarg.name.name: kwarg.value(env) for kwarg in self.named} - try: - return function(*args, **kwargs) - except Exception as e: - env.errors.append(e) - return FluentNone(function_name + "()") + if isinstance(self.callee, (TermReference, AttributeExpression)): + term = lookup_reference(self.callee, env) + if args: + env.errors.append(FluentFormatError("Ignored positional arguments passed to term '{0}'" + .format(reference_to_id(self.callee)))) + with env.modified_for_term_reference(args=kwargs): + return term(env) + + # builtin or custom function call + function_name = self.callee.id.name + try: + function = env.context._functions[function_name] + except LookupError: + env.errors.append(FluentReferenceError("Unknown function: {0}" + .format(function_name))) + return FluentNone(function_name + "()") + + try: + return function(*args, **kwargs) + except Exception as e: + env.errors.append(e) + return FluentNoneResolver(function_name + "()") @handle.register(FluentNumber) From 577f38ff4e8de0db4b08473da9878f1e23ecc278 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 13:03:21 +0100 Subject: [PATCH 02/19] refactor lookup into bundle, change dirty, and make named arguments work --- fluent.runtime/fluent/runtime/__init__.py | 11 +++++++---- fluent.runtime/fluent/runtime/resolver.py | 24 +++++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index 98f0cdd1..cd3e8853 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -56,16 +56,19 @@ def has_message(self, message_id): return False return message_id in self._messages_and_terms + def lookup(self, full_id): + if full_id not in self._compiled: + message = self._messages_and_terms[full_id] + self._compiled[full_id] = self._compiler(message.value) + return self._compiled[full_id] + def format(self, message_id, args=None): if message_id.startswith(TERM_SIGIL): raise LookupError(message_id) if args is None: args = {} - if message_id not in self._compiled: - message = self._messages_and_terms[message_id] - self._compiled[message_id] = self._compiler(message.value) - resolve = self._compiled[message_id] errors = [] + resolve = self.lookup(message_id) env = ResolverEnvironment(context=self, current=CurrentEnvironment(args=args), errors=errors) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 919274a2..c30e35d4 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -9,7 +9,7 @@ from fluent.syntax import ast as FTL from .errors import FluentCyclicReferenceError, FluentFormatError, FluentReferenceError -from .types import FluentDateType, FluentNone, FluentNumber, fluent_date, fluent_number +from .types import FluentType, FluentDateType, FluentNone, FluentNumber, fluent_date, fluent_number from .utils import numeric_to_native, reference_to_id, unknown_reference_error_obj try: @@ -54,7 +54,6 @@ class CurrentEnvironment(object): class ResolverEnvironment(object): context = attr.ib() errors = attr.ib() - dirty = attr.ib(factory=set) part_count = attr.ib(default=0) current = attr.ib(factory=CurrentEnvironment) @@ -141,11 +140,16 @@ def __call__(self, env): return FluentNone() self.dirty = True retval = ''.join( - element(env) for element in self.elements + self.resolve(element(env), env) for element in self.elements ) self.dirty = False return retval + def resolve(self, fluentish, env): + if isinstance(fluentish, FluentType): + return fluentish.format(env.context._babel_locale) + return fluentish + class TextElement(FTL.TextElement, Literal): def __call__(self, env): @@ -189,9 +193,8 @@ def lookup_reference(ref, env): AST node, or FluentNone if not found, including fallback logic """ ref_id = reference_to_id(ref) - try: - return env.context._compiler(env.context._messages_and_terms[ref_id]) + return env.context.lookup(ref_id) except LookupError: env.errors.append(unknown_reference_error_obj(ref_id)) @@ -199,7 +202,7 @@ def lookup_reference(ref, env): # Fallback parent_id = reference_to_id(ref.ref) try: - return env.context._compiler(env.context._messages_and_terms[parent_id]) + return env.context.lookup(parent_id) except LookupError: # Don't add error here, because we already added error for the # actual thing we were looking for. @@ -330,10 +333,10 @@ def __call__(self, env): # TODO What to do if message is not a VariantList? # Need test at least. - assert isinstance(message.value, VariantList) + assert isinstance(message, VariantList) variant_name = self.key.name - return message.value.select_from_variant_list(env, variant_name) + return message.select_from_variant_list(env, variant_name) class CallExpression(FTL.CallExpression, BaseResolver): @@ -365,6 +368,11 @@ def __call__(self, env): return FluentNoneResolver(function_name + "()") +class NamedArgument(FTL.NamedArgument, BaseResolver): + def __call__(self, env): + return self.value(env) + + @handle.register(FluentNumber) def handle_fluent_number(number, env): return number.format(env.context._babel_locale) From 0e8919919ab1eb891828b996fd44db9e08c53937 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 13:16:52 +0100 Subject: [PATCH 03/19] Fix VariableReference --- fluent.runtime/fluent/runtime/resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index c30e35d4..1700d5c9 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -226,10 +226,10 @@ def __call__(self, env): (int, float, Decimal, date, datetime, text_type)): - return lambda env: arg_val + return arg_val env.errors.append(TypeError("Unsupported external type: {0}, {1}" .format(name, type(arg_val)))) - return FluentNoneResolver(name) + return FluentNone(name) class AttributeExpression(FTL.AttributeExpression, BaseResolver): From 849061ce293b6181a705694de3b6bff36e678808 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 16:13:42 +0100 Subject: [PATCH 04/19] Fix numbers --- fluent.runtime/fluent/runtime/resolver.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 1700d5c9..10dc3211 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -9,7 +9,7 @@ from fluent.syntax import ast as FTL from .errors import FluentCyclicReferenceError, FluentFormatError, FluentReferenceError -from .types import FluentType, FluentDateType, FluentNone, FluentNumber, fluent_date, fluent_number +from .types import FluentType, FluentDateType, FluentNone, FluentNumber, FluentInt, FluentFloat, fluent_date, fluent_number from .utils import numeric_to_native, reference_to_id, unknown_reference_error_obj try: @@ -146,7 +146,7 @@ def __call__(self, env): return retval def resolve(self, fluentish, env): - if isinstance(fluentish, FluentType): + if isinstance(fluentish, (FluentType, FluentNumber)): return fluentish.format(env.context._babel_locale) return fluentish @@ -166,7 +166,13 @@ def __call__(self, env): return self.value -class NumberLiteral(FTL.NumberLiteral, Literal): +class NumberLiteral(FTL.NumberLiteral, BaseResolver): + def __init__(self, value, **kwargs): + super(NumberLiteral, self).__init__(value, **kwargs) + if '.' in self.value: + self.value = FluentFloat(self.value) + else: + self.value = FluentInt(self.value) def __call__(self, env): return self.value From e2f839531815d08e18f1fcd73f4c0cb17e9ffb87 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 16:26:17 +0100 Subject: [PATCH 05/19] use isolation --- fluent.runtime/fluent/runtime/__init__.py | 2 +- fluent.runtime/fluent/runtime/prepare.py | 11 ++++++++++- fluent.runtime/fluent/runtime/resolver.py | 16 +++++++++++----- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index cd3e8853..c7dc63a3 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -35,7 +35,7 @@ def __init__(self, locales, functions=None, use_isolating=True): self._use_isolating = use_isolating self._messages_and_terms = {} self._compiled = {} - self._compiler = Compiler() + self._compiler = Compiler(use_isolating=use_isolating) self._babel_locale = self._get_babel_locale() self._plural_form = babel.plural.to_python(self._babel_locale.plural_form) diff --git a/fluent.runtime/fluent/runtime/prepare.py b/fluent.runtime/fluent/runtime/prepare.py index 30139f9a..90cb3b85 100644 --- a/fluent.runtime/fluent/runtime/prepare.py +++ b/fluent.runtime/fluent/runtime/prepare.py @@ -4,8 +4,9 @@ class Compiler(object): - def __init__(self, bundle=None): + def __init__(self, use_isolating=False, bundle=None): self.bundle = None + self.use_isolating = use_isolating def __call__(self, item): if isinstance(item, FTL.BaseNode): @@ -28,11 +29,19 @@ def compile_generic(self, nodename, **kwargs): return getattr(resolver, nodename)(**kwargs) def compile_Placeable(self, _, expression, **kwargs): + if self.use_isolating: + return resolver.IsolatingPlaceable(expression=expression, **kwargs) if isinstance(expression, resolver.Literal): return expression return resolver.Placeable(expression=expression, **kwargs) def compile_Pattern(self, _, elements, **kwargs): + if ( + len(elements) == 1 and + isinstance(elements[0], resolver.IsolatingPlaceable) + ): + # Don't isolate isolated placeables + return elements[0].expression if any( not isinstance(child, resolver.Literal) for child in elements diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 10dc3211..c95326b1 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -140,15 +140,15 @@ def __call__(self, env): return FluentNone() self.dirty = True retval = ''.join( - self.resolve(element(env), env) for element in self.elements + resolve(element(env), env) for element in self.elements ) self.dirty = False return retval - def resolve(self, fluentish, env): - if isinstance(fluentish, (FluentType, FluentNumber)): - return fluentish.format(env.context._babel_locale) - return fluentish +def resolve(fluentish, env): + if isinstance(fluentish, (FluentType, FluentNumber)): + return fluentish.format(env.context._babel_locale) + return fluentish class TextElement(FTL.TextElement, Literal): @@ -161,6 +161,12 @@ def __call__(self, env): return self.expression(env) +class IsolatingPlaceable(FTL.Placeable, BaseResolver): + def __call__(self, env): + inner = self.expression(env) + return "\u2068" + resolve(inner, env) + "\u2069" + + class StringLiteral(FTL.StringLiteral, Literal): def __call__(self, env): return self.value From 9a753569637c622c295f2f1a7e9d5a02c40a37c5 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 17:32:08 +0100 Subject: [PATCH 06/19] fix external variables to be fluent types --- fluent.runtime/fluent/runtime/__init__.py | 11 +++++++---- fluent.runtime/fluent/runtime/resolver.py | 2 +- fluent.runtime/fluent/runtime/types.py | 4 ++-- fluent.runtime/fluent/runtime/utils.py | 23 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index c7dc63a3..1fd282eb 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -10,7 +10,7 @@ from .builtins import BUILTINS from .prepare import Compiler from .resolver import ResolverEnvironment, CurrentEnvironment -from .utils import ATTRIBUTE_SEPARATOR, TERM_SIGIL, add_message_and_attrs_to_store, ast_to_id +from .utils import ATTRIBUTE_SEPARATOR, TERM_SIGIL, add_message_and_attrs_to_store, ast_to_id, native_to_fluent class FluentBundle(object): @@ -65,12 +65,15 @@ def lookup(self, full_id): def format(self, message_id, args=None): if message_id.startswith(TERM_SIGIL): raise LookupError(message_id) - if args is None: - args = {} + fluent_args = {} + if args is not None: + for argname, argvalue in args.items(): + fluent_args[argname] = native_to_fluent(argvalue) + errors = [] resolve = self.lookup(message_id) env = ResolverEnvironment(context=self, - current=CurrentEnvironment(args=args), + current=CurrentEnvironment(args=fluent_args), errors=errors) return [resolve(env), errors] diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index c95326b1..735f9773 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -146,7 +146,7 @@ def __call__(self, env): return retval def resolve(fluentish, env): - if isinstance(fluentish, (FluentType, FluentNumber)): + if isinstance(fluentish, FluentType): return fluentish.format(env.context._babel_locale) return fluentish diff --git a/fluent.runtime/fluent/runtime/types.py b/fluent.runtime/fluent/runtime/types.py index abf631bd..c8d26e38 100644 --- a/fluent.runtime/fluent/runtime/types.py +++ b/fluent.runtime/fluent/runtime/types.py @@ -81,7 +81,7 @@ class NumberFormatOptions(object): maximumSignificantDigits = attr.ib(default=None) -class FluentNumber(object): +class FluentNumber(FluentType): default_number_format_options = NumberFormatOptions() @@ -276,7 +276,7 @@ class DateFormatOptions(object): _SUPPORTED_DATETIME_OPTIONS = ['dateStyle', 'timeStyle', 'timeZone'] -class FluentDateType(object): +class FluentDateType(FluentType): # We need to match signature of `__init__` and `__new__` due to the way # some Python implementation (e.g. PyPy) implement some methods. # So we leave those alone, and implement another `_init_options` diff --git a/fluent.runtime/fluent/runtime/utils.py b/fluent.runtime/fluent/runtime/utils.py index e6f793bd..854282fb 100644 --- a/fluent.runtime/fluent/runtime/utils.py +++ b/fluent.runtime/fluent/runtime/utils.py @@ -1,5 +1,11 @@ +from __future__ import absolute_import, unicode_literals + +from datetime import date, datetime +from decimal import Decimal + from fluent.syntax.ast import AttributeExpression, Term, TermReference +from .types import FluentInt, FluentFloat, FluentDecimal, FluentDate, FluentDateTime from .errors import FluentReferenceError TERM_SIGIL = '-' @@ -37,6 +43,23 @@ def numeric_to_native(val): return int(val) +def native_to_fluent(val): + """ + Convert a python type to a Fluent Type. + """ + if isinstance(val, int): + return FluentInt(val) + if isinstance(val, float): + return FluentFloat(val) + if isinstance(val, Decimal): + return FluentDecimal(val) + + if isinstance(val, date): + return FluentDate.from_date(val) + if isinstance(val, datetime): + return FluentDateTime.from_date(val) + return val + def reference_to_id(ref): """ Returns a string reference for a MessageReference, TermReference or AttributeExpression From ef8972f06787a354cba3da9aea590e71564e5ca6 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 18:58:10 +0100 Subject: [PATCH 07/19] fix attributes and value-less messages --- fluent.runtime/fluent/runtime/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index 1fd282eb..3b426107 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -58,8 +58,13 @@ def has_message(self, message_id): def lookup(self, full_id): if full_id not in self._compiled: - message = self._messages_and_terms[full_id] - self._compiled[full_id] = self._compiler(message.value) + entry_id = full_id.split(ATTRIBUTE_SEPARATOR, 1)[0] + entry = self._messages_and_terms[entry_id] + compiled = self._compiler(entry) + if compiled.value is not None: + self._compiled[entry_id] = compiled.value + for attr in compiled.attributes: + self._compiled[ATTRIBUTE_SEPARATOR.join([entry_id, attr.id.name])] = attr.value return self._compiled[full_id] def format(self, message_id, args=None): From b3423f3b8cae797630bb7690b0fe08b2d9b8fdba Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 19:31:13 +0100 Subject: [PATCH 08/19] Add protection against abuse. This changes a test, I don't think that the returned value should contain any non-source text fragments from a FluentNone. --- fluent.runtime/fluent/runtime/resolver.py | 14 +++++++++++++- fluent.runtime/tests/test_bomb.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 735f9773..09632adf 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -138,16 +138,28 @@ def __call__(self, env): if self.dirty: env.errors.append(FluentCyclicReferenceError("Cyclic reference")) return FluentNone() + if env.part_count > MAX_PARTS: + return "" self.dirty = True + elements = self.elements + remaining_parts = MAX_PARTS - env.part_count + if len(self.elements) > remaining_parts: + elements = elements[:remaining_parts + 1] + env.errors.append(ValueError("Too many parts in message (> {0}), " + "aborting.".format(MAX_PARTS))) retval = ''.join( - resolve(element(env), env) for element in self.elements + resolve(element(env), env) for element in elements ) + env.part_count += len(elements) self.dirty = False return retval def resolve(fluentish, env): if isinstance(fluentish, FluentType): return fluentish.format(env.context._babel_locale) + if isinstance(fluentish, six.string_types): + if len(fluentish) > MAX_PART_LENGTH: + return fluentish[:MAX_PART_LENGTH] return fluentish diff --git a/fluent.runtime/tests/test_bomb.py b/fluent.runtime/tests/test_bomb.py index 889cb5a0..082de96b 100644 --- a/fluent.runtime/tests/test_bomb.py +++ b/fluent.runtime/tests/test_bomb.py @@ -39,5 +39,5 @@ def test_max_expansions_protection(self): # Without protection, emptylolz will take a really long time to # evaluate, although it generates an empty message. val, errs = self.ctx.format('emptylolz') - self.assertEqual(val, '???') + self.assertEqual(val, '') self.assertEqual(len(errs), 1) From 4eeeba7be5a9e954ae1c021b50a26d80b2a589f3 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 20:53:09 +0100 Subject: [PATCH 09/19] remove unused code --- fluent.runtime/fluent/runtime/resolver.py | 95 ++--------------------- fluent.runtime/setup.py | 8 +- 2 files changed, 6 insertions(+), 97 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 09632adf..81a32d18 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -12,12 +12,6 @@ from .types import FluentType, FluentDateType, FluentNone, FluentNumber, FluentInt, FluentFloat, fluent_date, fluent_number from .utils import numeric_to_native, reference_to_id, unknown_reference_error_obj -try: - from functools import singledispatch -except ImportError: - # Python < 3.4 - from singledispatch import singledispatch - text_type = six.text_type @@ -75,41 +69,6 @@ def modified_for_term_reference(self, args=None): error_for_missing_arg=False) -def resolve(context, message, args): - """ - Given a FluentBundle, a Message instance and some arguments, - resolve the message to a string. - - This is the normal entry point for this module. - """ - errors = [] - env = ResolverEnvironment(context=context, - current=CurrentEnvironment(args=args), - errors=errors) - return fully_resolve(message, env), errors - - -def fully_resolve(expr, env): - """ - Fully resolve an expression to a string - """ - # This differs from 'handle' in that 'handle' will often return non-string - # objects, even if a string could have been returned, to allow for further - # handling of that object e.g. attributes of messages. fully_resolve is - # only used when we must have a string. - retval = handle(expr, env) - if isinstance(retval, text_type): - return retval - - return fully_resolve(retval, env) - - -@singledispatch -def handle(expr, env): - raise NotImplementedError("Cannot handle object of type {0}" - .format(type(expr).__name__)) - - class BaseResolver(object): def __call__(self, env): raise NotImplementedError @@ -120,13 +79,11 @@ class Literal(BaseResolver): class Message(FTL.Message, BaseResolver): - def __call__(self, env): - return self.value(env) + pass class Term(FTL.Term, BaseResolver): - def __call__(self, env): - return self.value(env) + pass class Pattern(FTL.Pattern, BaseResolver): @@ -262,8 +219,7 @@ def __call__(self, env): class Attribute(FTL.Attribute, BaseResolver): - def __call__(self, env): - return self.value(env) + pass class VariantListResolver(BaseResolver): @@ -315,8 +271,6 @@ def select_from_select_expression(self, env, key): if found is None: found = default - if found is None: - return FluentNoneResolver() return found.value(env) @@ -340,8 +294,7 @@ def match(val1, val2, env): class Variant(FTL.Variant, BaseResolver): - def __call__(self, env): - return self.value(env) + pass class Identifier(FTL.Identifier, BaseResolver): @@ -352,8 +305,6 @@ def __call__(self, env): class VariantExpression(FTL.VariantExpression, BaseResolver): def __call__(self, env): message = lookup_reference(self.ref, env) - if isinstance(message, FluentNone): - return FluentNoneResolver(FluentNone.name) # TODO What to do if message is not a VariantList? # Need test at least. @@ -393,40 +344,4 @@ def __call__(self, env): class NamedArgument(FTL.NamedArgument, BaseResolver): - def __call__(self, env): - return self.value(env) - - -@handle.register(FluentNumber) -def handle_fluent_number(number, env): - return number.format(env.context._babel_locale) - - -@handle.register(int) -def handle_int(integer, env): - return fluent_number(integer).format(env.context._babel_locale) - - -@handle.register(float) -def handle_float(f, env): - return fluent_number(f).format(env.context._babel_locale) - - -@handle.register(Decimal) -def handle_decimal(d, env): - return fluent_number(d).format(env.context._babel_locale) - - -@handle.register(FluentDateType) -def handle_fluent_date_type(d, env): - return d.format(env.context._babel_locale) - - -@handle.register(date) -def handle_date(d, env): - return fluent_date(d).format(env.context._babel_locale) - - -@handle.register(datetime) -def handle_datetime(d, env): - return fluent_date(d).format(env.context._babel_locale) + pass diff --git a/fluent.runtime/setup.py b/fluent.runtime/setup.py index 7acc0802..04338fd9 100755 --- a/fluent.runtime/setup.py +++ b/fluent.runtime/setup.py @@ -2,12 +2,6 @@ from setuptools import setup import sys -if sys.version_info < (3, 4): - extra_requires = ['singledispatch>=3.4'] -else: - # functools.singledispatch is in stdlib from Python 3.4 onwards. - extra_requires = [] - setup(name='fluent.runtime', version='0.1', description='Localization library for expressive translations.', @@ -30,7 +24,7 @@ 'attrs', 'babel', 'pytz', - ] + extra_requires, + ], tests_require=['six'], test_suite='tests' ) From b490f5b396e8e2d0b9f9d2ffd96555346b3abdf1 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 20:55:11 +0100 Subject: [PATCH 10/19] fix datetime --- fluent.runtime/fluent/runtime/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fluent.runtime/fluent/runtime/utils.py b/fluent.runtime/fluent/runtime/utils.py index 854282fb..f8c4571e 100644 --- a/fluent.runtime/fluent/runtime/utils.py +++ b/fluent.runtime/fluent/runtime/utils.py @@ -54,10 +54,10 @@ def native_to_fluent(val): if isinstance(val, Decimal): return FluentDecimal(val) + if isinstance(val, datetime): + return FluentDateTime.from_date_time(val) if isinstance(val, date): return FluentDate.from_date(val) - if isinstance(val, datetime): - return FluentDateTime.from_date(val) return val def reference_to_id(ref): From 4cd29ff168d579d677e69869d1c57c9dd0678ccb Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 20:59:13 +0100 Subject: [PATCH 11/19] remove unused code, flake8 --- fluent.runtime/fluent/runtime/resolver.py | 6 ++++-- fluent.runtime/fluent/runtime/utils.py | 13 +------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 81a32d18..87663341 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -9,8 +9,8 @@ from fluent.syntax import ast as FTL from .errors import FluentCyclicReferenceError, FluentFormatError, FluentReferenceError -from .types import FluentType, FluentDateType, FluentNone, FluentNumber, FluentInt, FluentFloat, fluent_date, fluent_number -from .utils import numeric_to_native, reference_to_id, unknown_reference_error_obj +from .types import FluentType, FluentNone, FluentInt, FluentFloat +from .utils import reference_to_id, unknown_reference_error_obj text_type = six.text_type @@ -111,6 +111,7 @@ def __call__(self, env): self.dirty = False return retval + def resolve(fluentish, env): if isinstance(fluentish, FluentType): return fluentish.format(env.context._babel_locale) @@ -148,6 +149,7 @@ def __init__(self, value, **kwargs): self.value = FluentFloat(self.value) else: self.value = FluentInt(self.value) + def __call__(self, env): return self.value diff --git a/fluent.runtime/fluent/runtime/utils.py b/fluent.runtime/fluent/runtime/utils.py index f8c4571e..575511b4 100644 --- a/fluent.runtime/fluent/runtime/utils.py +++ b/fluent.runtime/fluent/runtime/utils.py @@ -31,18 +31,6 @@ def add_message_and_attrs_to_store(store, ref_id, item, is_parent=True): is_parent=False) -def numeric_to_native(val): - """ - Given a numeric string (as defined by fluent spec), - return an int or float - """ - # val matches this EBNF: - # '-'? [0-9]+ ('.' [0-9]+)? - if '.' in val: - return float(val) - return int(val) - - def native_to_fluent(val): """ Convert a python type to a Fluent Type. @@ -60,6 +48,7 @@ def native_to_fluent(val): return FluentDate.from_date(val) return val + def reference_to_id(ref): """ Returns a string reference for a MessageReference, TermReference or AttributeExpression From 67a2b3608fdd6c3f7f5d8637014e58d3a049b21b Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 21:41:00 +0100 Subject: [PATCH 12/19] We expand attributes at compile time --- fluent.runtime/fluent/runtime/__init__.py | 6 ++---- fluent.runtime/fluent/runtime/utils.py | 10 ---------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index 3b426107..048cf08e 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -10,7 +10,7 @@ from .builtins import BUILTINS from .prepare import Compiler from .resolver import ResolverEnvironment, CurrentEnvironment -from .utils import ATTRIBUTE_SEPARATOR, TERM_SIGIL, add_message_and_attrs_to_store, ast_to_id, native_to_fluent +from .utils import ATTRIBUTE_SEPARATOR, TERM_SIGIL, ast_to_id, native_to_fluent class FluentBundle(object): @@ -47,9 +47,7 @@ def add_messages(self, source): if isinstance(item, (Message, Term)): full_id = ast_to_id(item) if full_id not in self._messages_and_terms: - # We add attributes to the store to enable faster looker - # later, and more direct code in some instances. - add_message_and_attrs_to_store(self._messages_and_terms, full_id, item) + self._messages_and_terms[full_id] = item def has_message(self, message_id): if message_id.startswith(TERM_SIGIL) or ATTRIBUTE_SEPARATOR in message_id: diff --git a/fluent.runtime/fluent/runtime/utils.py b/fluent.runtime/fluent/runtime/utils.py index 575511b4..47a67fdd 100644 --- a/fluent.runtime/fluent/runtime/utils.py +++ b/fluent.runtime/fluent/runtime/utils.py @@ -21,16 +21,6 @@ def ast_to_id(ast): return ast.id.name -def add_message_and_attrs_to_store(store, ref_id, item, is_parent=True): - store[ref_id] = item - if is_parent: - for attr in item.attributes: - add_message_and_attrs_to_store(store, - _make_attr_id(ref_id, attr.id.name), - attr, - is_parent=False) - - def native_to_fluent(val): """ Convert a python type to a Fluent Type. From ec7b73229e40c9cdc2076c715566c415f42d90fa Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 21:43:14 +0100 Subject: [PATCH 13/19] We don't use bundle in the compiler --- fluent.runtime/fluent/runtime/prepare.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fluent.runtime/fluent/runtime/prepare.py b/fluent.runtime/fluent/runtime/prepare.py index 90cb3b85..698a5dd9 100644 --- a/fluent.runtime/fluent/runtime/prepare.py +++ b/fluent.runtime/fluent/runtime/prepare.py @@ -4,8 +4,7 @@ class Compiler(object): - def __init__(self, use_isolating=False, bundle=None): - self.bundle = None + def __init__(self, use_isolating=False): self.use_isolating = use_isolating def __call__(self, item): From 6d6c21aade26c960e3f793b20337043299d08ba5 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 22:03:06 +0100 Subject: [PATCH 14/19] simplify VariantList and VariantExpression --- fluent.runtime/fluent/runtime/resolver.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 87663341..7bd83d4b 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -224,8 +224,8 @@ class Attribute(FTL.Attribute, BaseResolver): pass -class VariantListResolver(BaseResolver): - def select_from_variant_list(self, env, key): +class VariantList(FTL.VariantList, BaseResolver): + def __call__(self, env, key=None): found = None for variant in self.variants: if variant.default: @@ -244,17 +244,11 @@ def select_from_variant_list(self, env, key): env.errors.append(FluentReferenceError("Unknown variant: {0}" .format(key))) found = default - if found is None: - return FluentNoneResolver() + assert found, "Not having a default variant is a parse error" return found.value(env) -class VariantList(FTL.VariantList, VariantListResolver): - def __call__(self, env): - return self.select_from_variant_list(env, None) - - class SelectExpression(FTL.SelectExpression, BaseResolver): def __call__(self, env): key = self.selector(env) @@ -313,7 +307,7 @@ def __call__(self, env): assert isinstance(message, VariantList) variant_name = self.key.name - return message.select_from_variant_list(env, variant_name) + return message(env, variant_name) class CallExpression(FTL.CallExpression, BaseResolver): From 5172851c8fdf1b7cd1c7b18222f0506e90aa25f7 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 22:24:23 +0100 Subject: [PATCH 15/19] self-review nits on VariableReference. I wonder if we should throw on bad external variables instead of making them a silent runtime error. --- fluent.runtime/fluent/runtime/resolver.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 7bd83d4b..86b7a0f9 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -194,7 +194,7 @@ def lookup_reference(ref, env): return FluentNoneResolver(ref_id) -class VariableReference(FTL.VariableReference): +class VariableReference(FTL.VariableReference, BaseResolver): def __call__(self, env): name = self.id.name try: @@ -205,10 +205,7 @@ def __call__(self, env): FluentReferenceError("Unknown external: {0}".format(name))) return FluentNoneResolver(name) - if isinstance(arg_val, - (int, float, Decimal, - date, datetime, - text_type)): + if isinstance(arg_val, (FluentType, text_type)): return arg_val env.errors.append(TypeError("Unsupported external type: {0}, {1}" .format(name, type(arg_val)))) From 3ebafce762fa13532fe631b1eeb348ba01291351 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 5 Feb 2019 22:30:20 +0100 Subject: [PATCH 16/19] comment --- fluent.runtime/fluent/runtime/resolver.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 86b7a0f9..e107ccd0 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -70,6 +70,14 @@ def modified_for_term_reference(self, args=None): class BaseResolver(object): + """ + Abstract base class of all partially evaluated resolvers. + + Subclasses should implement __call__, with a + ResolverEnvironment as parameter. An exception are wrapper + classes that don't show up in the evaluation, but need to + be part of the compiled tree structure. + """ def __call__(self, env): raise NotImplementedError From 584723c1a1f5ef1d54698e0059e490cab3022760 Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Mon, 18 Feb 2019 18:20:13 +0100 Subject: [PATCH 17/19] update dependencies --- fluent.runtime/setup.py | 2 +- fluent.runtime/tox.ini | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fluent.runtime/setup.py b/fluent.runtime/setup.py index 04338fd9..d22bb27c 100755 --- a/fluent.runtime/setup.py +++ b/fluent.runtime/setup.py @@ -20,7 +20,7 @@ ], packages=['fluent', 'fluent.runtime'], install_requires=[ - 'fluent.syntax>=0.10,<=0.11', + 'fluent.syntax>=0.12,<=0.13', 'attrs', 'babel', 'pytz', diff --git a/fluent.runtime/tox.ini b/fluent.runtime/tox.ini index 8fd96433..68361a9e 100644 --- a/fluent.runtime/tox.ini +++ b/fluent.runtime/tox.ini @@ -6,10 +6,8 @@ skipsdist=True setenv = PYTHONPATH = {toxinidir} deps = - fluent.syntax>=0.10,<=0.11 + fluent.syntax>=0.12,<=0.13 six attrs Babel - py27: singledispatch - pypy: singledispatch commands = ./runtests.py From 32f1cb7b0a1e9b2bad88524d3c0518da2244255f Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Mon, 18 Feb 2019 20:40:06 +0100 Subject: [PATCH 18/19] use dict comprehension --- fluent.runtime/fluent/runtime/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index 048cf08e..284b0eb4 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -68,10 +68,13 @@ def lookup(self, full_id): def format(self, message_id, args=None): if message_id.startswith(TERM_SIGIL): raise LookupError(message_id) - fluent_args = {} if args is not None: - for argname, argvalue in args.items(): - fluent_args[argname] = native_to_fluent(argvalue) + fluent_args = { + argname: native_to_fluent(argvalue) + for argname, argvalue in args.items() + } + else: + fluent_args = {} errors = [] resolve = self.lookup(message_id) From ea239e1311fc3e4f105f0f9104074f50412cb96a Mon Sep 17 00:00:00 2001 From: Axel Hecht Date: Tue, 19 Feb 2019 11:19:09 +0100 Subject: [PATCH 19/19] Clean up module namespace in resolver, add docstring to clarify what the module is doing --- fluent.runtime/fluent/runtime/resolver.py | 34 ++++++++++++++--------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index e107ccd0..1b41e450 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -13,18 +13,23 @@ from .utils import reference_to_id, unknown_reference_error_obj -text_type = six.text_type +""" +The classes in this module are used to transform the source +AST to a partially evaluated resolver tree. They're subclasses +to the syntax AST node, and `BaseResolver`. Syntax nodes that +don't require special handling, but have children that need to be +transformed, need to just inherit from their syntax base class and +`BaseResolver`. When adding to the module namespace here, watch +out for naming conflicts with `fluent.syntax.ast`. -# Prevent expansion of too long placeables, for memory DOS protection -MAX_PART_LENGTH = 2500 - -# Prevent messages with too many sub parts, for CPI DOS protection -MAX_PARTS = 1000 +`ResolverEnvironment` is the `env` passed to the `__call__` method +in the resolver tree. The `CurrentEnvironment` keeps track of the +modifyable state in the resolver environment. +""" -# Unicode bidi isolation characters. -FSI = "\u2068" -PDI = "\u2069" +# Prevent expansion of too long placeables, for memory DOS protection +MAX_PART_LENGTH = 2500 @attr.s @@ -95,6 +100,9 @@ class Term(FTL.Term, BaseResolver): class Pattern(FTL.Pattern, BaseResolver): + # Prevent messages with too many sub parts, for CPI DOS protection + MAX_PARTS = 1000 + def __init__(self, *args, **kwargs): super(Pattern, self).__init__(*args, **kwargs) self.dirty = False @@ -103,15 +111,15 @@ def __call__(self, env): if self.dirty: env.errors.append(FluentCyclicReferenceError("Cyclic reference")) return FluentNone() - if env.part_count > MAX_PARTS: + if env.part_count > self.MAX_PARTS: return "" self.dirty = True elements = self.elements - remaining_parts = MAX_PARTS - env.part_count + remaining_parts = self.MAX_PARTS - env.part_count if len(self.elements) > remaining_parts: elements = elements[:remaining_parts + 1] env.errors.append(ValueError("Too many parts in message (> {0}), " - "aborting.".format(MAX_PARTS))) + "aborting.".format(self.MAX_PARTS))) retval = ''.join( resolve(element(env), env) for element in elements ) @@ -213,7 +221,7 @@ def __call__(self, env): FluentReferenceError("Unknown external: {0}".format(name))) return FluentNoneResolver(name) - if isinstance(arg_val, (FluentType, text_type)): + if isinstance(arg_val, (FluentType, six.text_type)): return arg_val env.errors.append(TypeError("Unsupported external type: {0}, {1}" .format(name, type(arg_val))))