Skip to content

fix(3259): Support deserialization of Locales created using BCP 47 format #3265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 8, 2021
+237 −1
Merged

fix(3259): Support deserialization of Locales created using BCP 47 format #3265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18c2d86
fix(3259): Fix to support deserialization of Locales created using BC…
praneethjreddy Aug 31, 2021
2a682c0
fix(3259): Adding test case for special cases ja_JP_JP and th_TH_TH
praneethjreddy Aug 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion src/main/java/com/fasterxml/jackson/databind/deser/std/FromStringDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Currency;
import java.util.IllformedLocaleException;
import java.util.Locale;
import java.util.TimeZone;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -326,7 +327,12 @@ protected Object _deserialize(String value, DeserializationContext ctxt) throws
return new Locale(first, value);
}
String second = value.substring(0, ix);
return new Locale(first, second, value.substring(ix+1));
if(!_isScriptOrExtensionPresent(value)) {
return new Locale(first, second, value.substring(ix+1));
} else {
// Issue #3259: Support for BCP 47 java.util.Locale Serialization / De-serialization
return _deSerializeBCP47Locale(value, ix, first, second);
}
}
case STD_CHARSET:
return Charset.forName(value);
Expand Down Expand Up @@ -395,6 +401,45 @@ protected int _firstHyphenOrUnderscore(String str)
}
return -1;
}

private Locale _deSerializeBCP47Locale(String value, int ix, String first, String second) {
String third = "";
try {
int scriptExpIx = value.indexOf("_#");
/*
* Below condition checks if variant value is present to handle empty variant values such as
* en__#Latn_x-ext
* _US_#Latn
*/
if (scriptExpIx > 0 && scriptExpIx > ix)
third = value.substring(ix + 1, scriptExpIx);
value = value.substring(scriptExpIx + 2);

if (value.indexOf('_') < 0 && value.indexOf('-') < 0) {
return new Locale.Builder().setLanguage(first)
.setRegion(second).setVariant(third).setScript(value).build();
}
if (value.indexOf('_') < 0) {
ix = value.indexOf('-');
return new Locale.Builder().setLanguage(first)
.setRegion(second).setVariant(third)
.setExtension(value.charAt(0), value.substring(ix + 1))
.build();
}
ix = value.indexOf('_');
return new Locale.Builder().setLanguage(first)
.setRegion(second).setVariant(third)
.setScript(value.substring(0, ix))
.setExtension(value.charAt(ix + 1), value.substring(ix + 3))
.build();
} catch(IllformedLocaleException ex) {
return new Locale(first, second, third);
}
}

private boolean _isScriptOrExtensionPresent(String value) {
return value.contains("_#");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this separator/marker documented somewhere? Would be great to have a link to explanation.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://docs.oracle.com/javase/7/docs/api/java/util/Locale.html#toString()

}
}

// @since 2.12 to simplify logic a bit: should not use coercions when reading
Expand Down
191 changes: 191 additions & 0 deletions src/test/java/com/fasterxml/jackson/databind/deser/std/FromStringDeserializerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
package com.fasterxml.jackson.databind.deser.std;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.BaseTest;
import com.fasterxml.jackson.databind.ObjectMapper;

import java.util.Locale;

public class FromStringDeserializerTest extends BaseTest {

private final Locale[] LOCALES = new Locale[]
{Locale.CANADA, Locale.ROOT, Locale.GERMAN, Locale.CHINESE, Locale.KOREA, Locale.TAIWAN};
private final ObjectMapper MAPPER = new ObjectMapper();

public void testLocaleDeserializeNonBCPFormat() throws JsonProcessingException {
Locale locale = new Locale("en", "US");
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);

locale = new Locale("en");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);

locale = new Locale("en", "US", "VARIANT");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);

locale = new Locale("en", "", "VARIANT");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);

locale = new Locale("", "US", "VARIANT");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);

locale = new Locale("", "US", "");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);
}

public void testLocaleDeserializeWithScript() throws JsonProcessingException {
Locale locale = new Locale.Builder().setLanguage("en").setRegion("GB").setVariant("VARIANT")
.setScript("Latn").build();
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("en").setScript("Latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setRegion("IN").setScript("Latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("fr").setRegion("CA").setScript("Latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setRegion("CA").setVariant("VARIANT").setScript("Latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("it").setVariant("VARIANT").setScript("Latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);
}

public void testLocaleDeserializeWithExtension() throws JsonProcessingException {
Locale locale = new Locale.Builder().setLanguage("en").setRegion("GB").setVariant("VARIANT")
.setExtension('x', "dummy").build();
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithExtension(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("en").setExtension('x', "dummy").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setRegion("IN").setExtension('x', "dummy").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("fr").setRegion("CA").setExtension('x', "dummy").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setRegion("CA").setVariant("VARIANT").setExtension('x', "dummy").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("it").setVariant("VARIANT").setExtension('x', "dummy").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocaleWithScript(locale, deSerializedLocale);
}

public void testLocaleDeserializeWithScriptAndExtension() throws JsonProcessingException {
Locale locale = new Locale.Builder().setLanguage("en").setRegion("GB").setVariant("VARIANT")
.setExtension('x', "dummy").setScript("latn").build();
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("en").setExtension('x', "dummy").setScript("latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = new Locale.Builder().setRegion("IN").setExtension('x', "dummy").setScript("latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("fr").setRegion("CA")
.setExtension('x', "dummy").setScript("latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = new Locale.Builder().setRegion("CA").setVariant("VARIANT")
.setExtension('x', "dummy").setScript("latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = new Locale.Builder().setLanguage("it").setVariant("VARIANT")
.setExtension('x', "dummy").setScript("latn").build();
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);
}

public void testLocaleDeserializeWithLanguageTag() throws JsonProcessingException {
Locale locale = Locale.forLanguageTag("en-US-x-debug");
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = Locale.forLanguageTag("en-US-x-lvariant-POSIX");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = Locale.forLanguageTag("de-POSIX-x-URP-lvariant-AbcDef");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);

locale = Locale.forLanguageTag("ar-aao");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = Locale.forLanguageTag("en-abc-def-us");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);
}

public void testIllFormedVariant() throws JsonProcessingException {
Locale locale = Locale.forLanguageTag("de-POSIX-x-URP-lvariant-Abc-Def");
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertBaseValues(locale, deSerializedLocale);
}

public void testLocaleDeserializeWithLocaleConstants() throws JsonProcessingException {
for (Locale locale: LOCALES) {
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);
}
}

public void testSpecialCases() throws JsonProcessingException {
Locale locale = new Locale("ja", "JP", "JP");
Locale deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);

locale = new Locale("th", "TH", "TH");
deSerializedLocale = MAPPER.readValue(MAPPER.writeValueAsString(locale), Locale.class);
assertLocale(locale, deSerializedLocale);
}

private void assertBaseValues(Locale expected, Locale actual) {
assertEquals(expected.getLanguage(), actual.getLanguage());
assertEquals(expected.getCountry(), actual.getCountry());
assertEquals(expected.getVariant(), actual.getVariant());
}

private void assertLocaleWithScript(Locale expected, Locale actual) {
assertBaseValues(expected, actual);
assertEquals(expected.getScript(), actual.getScript());
}

private void assertLocaleWithExtension(Locale expected, Locale actual) {
assertBaseValues(expected, actual);
assertEquals(expected.getExtension('x'), actual.getExtension('x'));
}

private void assertLocale(Locale expected, Locale actual) {
assertBaseValues(expected, actual);
assertEquals(expected.getExtension('x'), actual.getExtension('x'));
assertEquals(expected.getScript(), actual.getScript());
}
}