Skip to content

Commit 1151edf

Browse files
virajjasanivirajjasani
authored
HADOOP-17956. Replace all default Charset usage with UTF-8 (apache#3529)
Signed-off-by: Akira Ajisaka <[email protected]>
1 parent 7279fe8 commit 1151edf

File tree

34 files changed

+92
-124
lines changed

34 files changed

+92
-124
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
import java.io.PrintWriter;
4747
import java.lang.reflect.InvocationTargetException;
4848
import java.net.InetAddress;
49-
import java.nio.charset.Charset;
49+
import java.nio.charset.StandardCharsets;
5050
import java.nio.file.Files;
5151
import java.security.NoSuchAlgorithmException;
5252
import java.util.ArrayList;
@@ -924,7 +924,7 @@ private void printEnv(String variable) {
924924
*/
925925
private void dump(File file) throws IOException {
926926
try (InputStream in = Files.newInputStream(file.toPath())) {
927-
for (String line : IOUtils.readLines(in, Charset.defaultCharset())) {
927+
for (String line : IOUtils.readLines(in, StandardCharsets.UTF_8)) {
928928
println("%s", line);
929929
}
930930
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import java.net.URI;
2424
import java.net.URISyntaxException;
2525
import java.net.URL;
26-
import java.nio.charset.Charset;
26+
import java.nio.charset.StandardCharsets;
2727

2828
import org.apache.hadoop.classification.VisibleForTesting;
2929
import org.apache.commons.io.IOUtils;
@@ -225,8 +225,7 @@ public static char[] locatePassword(String envWithPass, String fileWithPass)
225225
throw new IOException("Password file does not exist");
226226
}
227227
try (InputStream is = pwdFile.openStream()) {
228-
pass = IOUtils.toString(is, Charset.defaultCharset()).trim()
229-
.toCharArray();
228+
pass = IOUtils.toString(is, StandardCharsets.UTF_8).trim().toCharArray();
230229
}
231230
}
232231
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import java.io.File;
2222
import java.io.IOException;
2323
import java.io.InputStreamReader;
24-
import java.nio.charset.Charset;
2524
import java.nio.charset.StandardCharsets;
2625
import java.nio.file.Files;
2726
import java.util.Collections;
@@ -223,8 +222,7 @@ public static boolean updateMapInternal(BiMap<Integer, String> map,
223222
Process process = Runtime.getRuntime().exec(
224223
new String[] { "bash", "-c", command });
225224
br = new BufferedReader(
226-
new InputStreamReader(process.getInputStream(),
227-
Charset.defaultCharset()));
225+
new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));
228226
String line = null;
229227
while ((line = br.readLine()) != null) {
230228
String[] nameId = line.split(regex);

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
import java.lang.reflect.Constructor;
3030
import java.lang.reflect.Field;
3131
import java.lang.reflect.Method;
32-
import java.nio.charset.Charset;
32+
import java.nio.charset.StandardCharsets;
3333
import java.util.ArrayList;
3434
import java.util.Arrays;
3535
import java.util.Comparator;
@@ -244,7 +244,7 @@ public static void logThreadInfo(Log log,
244244
try {
245245
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
246246
printThreadInfo(new PrintStream(buffer, false, "UTF-8"), title);
247-
log.info(buffer.toString(Charset.defaultCharset().name()));
247+
log.info(buffer.toString(StandardCharsets.UTF_8.name()));
248248
} catch (UnsupportedEncodingException ignored) {
249249
}
250250
}
@@ -273,7 +273,7 @@ public static void logThreadInfo(Logger log,
273273
try {
274274
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
275275
printThreadInfo(new PrintStream(buffer, false, "UTF-8"), title);
276-
log.info(buffer.toString(Charset.defaultCharset().name()));
276+
log.info(buffer.toString(StandardCharsets.UTF_8.name()));
277277
} catch (UnsupportedEncodingException ignored) {
278278
}
279279
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import java.io.IOException;
2424
import java.io.InputStreamReader;
2525
import java.io.InterruptedIOException;
26-
import java.nio.charset.Charset;
26+
import java.nio.charset.StandardCharsets;
2727
import java.util.Arrays;
2828
import java.util.Collections;
2929
import java.util.HashSet;
@@ -949,11 +949,11 @@ private void runCommand() throws IOException {
949949
timeOutTimer.schedule(timeoutTimerTask, timeOutInterval);
950950
}
951951
final BufferedReader errReader =
952-
new BufferedReader(new InputStreamReader(
953-
process.getErrorStream(), Charset.defaultCharset()));
952+
new BufferedReader(new InputStreamReader(process.getErrorStream(),
953+
StandardCharsets.UTF_8));
954954
BufferedReader inReader =
955-
new BufferedReader(new InputStreamReader(
956-
process.getInputStream(), Charset.defaultCharset()));
955+
new BufferedReader(new InputStreamReader(process.getInputStream(),
956+
StandardCharsets.UTF_8));
957957
final StringBuffer errMsg = new StringBuffer();
958958

959959
// read error and input streams as this would free up the buffers

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import java.io.StringWriter;
2828
import java.lang.reflect.Method;
2929
import java.net.URI;
30-
import java.nio.charset.Charset;
30+
import java.nio.charset.StandardCharsets;
3131
import java.nio.file.Files;
3232
import java.nio.file.Paths;
3333

@@ -125,7 +125,7 @@ public InputStream getInputStream(PathData item) throws IOException {
125125

126126
private String inputStreamToString(InputStream stream) throws IOException {
127127
StringWriter writer = new StringWriter();
128-
IOUtils.copy(stream, writer, Charset.defaultCharset());
128+
IOUtils.copy(stream, writer, StandardCharsets.UTF_8);
129129
return writer.toString();
130130
}
131131

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
import java.io.File;
3737
import java.io.FileInputStream;
3838
import java.io.IOException;
39-
import java.nio.charset.Charset;
39+
import java.nio.charset.StandardCharsets;
4040
import java.util.Properties;
4141
import java.util.concurrent.TimeUnit;
4242

@@ -236,7 +236,7 @@ public void testKeytabUnknownPrincipal() throws Throwable {
236236
*/
237237
private void dump(File file) throws IOException {
238238
try (FileInputStream in = new FileInputStream(file)) {
239-
for (String line : IOUtils.readLines(in, Charset.defaultCharset())) {
239+
for (String line : IOUtils.readLines(in, StandardCharsets.UTF_8)) {
240240
LOG.info(line);
241241
}
242242
}

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
import java.io.Writer;
6565
import java.net.HttpURLConnection;
6666
import java.net.URL;
67-
import java.nio.charset.Charset;
67+
import java.nio.charset.StandardCharsets;
6868
import java.security.Principal;
6969
import java.security.PrivilegedActionException;
7070
import java.security.PrivilegedExceptionAction;
@@ -555,8 +555,7 @@ public Void run() throws Exception {
555555
HttpURLConnection conn = aUrl.openConnection(url, token);
556556
Assert.assertEquals(HttpURLConnection.HTTP_OK,
557557
conn.getResponseCode());
558-
List<String> ret = IOUtils.readLines(conn.getInputStream(),
559-
Charset.defaultCharset());
558+
List<String> ret = IOUtils.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
560559
Assert.assertEquals(1, ret.size());
561560
Assert.assertEquals(FOO_USER, ret.get(0));
562561

@@ -626,8 +625,7 @@ public Void run() throws Exception {
626625
HttpURLConnection conn = aUrl.openConnection(url, token);
627626
Assert.assertEquals(HttpURLConnection.HTTP_OK,
628627
conn.getResponseCode());
629-
List<String> ret = IOUtils
630-
.readLines(conn.getInputStream(), Charset.defaultCharset());
628+
List<String> ret = IOUtils.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
631629
Assert.assertEquals(1, ret.size());
632630
Assert.assertEquals(FOO_USER, ret.get(0));
633631

@@ -851,15 +849,14 @@ public void testProxyUser() throws Exception {
851849
HttpURLConnection conn =
852850
(HttpURLConnection) new URL(strUrl).openConnection();
853851
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
854-
List<String> ret =
855-
IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset());
852+
List<String> ret = IOUtils.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
856853
Assert.assertEquals(1, ret.size());
857854
Assert.assertEquals(OK_USER, ret.get(0));
858855
strUrl = String.format("%s?user.name=%s&DOAS=%s", url.toExternalForm(),
859856
FOO_USER, OK_USER);
860857
conn = (HttpURLConnection) new URL(strUrl).openConnection();
861858
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
862-
ret = IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset());
859+
ret = IOUtils.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
863860
Assert.assertEquals(1, ret.size());
864861
Assert.assertEquals(OK_USER, ret.get(0));
865862

@@ -877,7 +874,7 @@ public Void run() throws Exception {
877874
Assert.assertEquals(HttpURLConnection.HTTP_OK,
878875
conn.getResponseCode());
879876
List<String> ret = IOUtils
880-
.readLines(conn.getInputStream(), Charset.defaultCharset());
877+
.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
881878
Assert.assertEquals(1, ret.size());
882879
Assert.assertEquals(OK_USER, ret.get(0));
883880

@@ -898,7 +895,7 @@ public Void run() throws Exception {
898895
Assert.assertEquals(HttpURLConnection.HTTP_OK,
899896
conn.getResponseCode());
900897
ret = IOUtils
901-
.readLines(conn.getInputStream(), Charset.defaultCharset());
898+
.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
902899
Assert.assertEquals(1, ret.size());
903900
Assert.assertEquals(FOO_USER, ret.get(0));
904901

@@ -960,7 +957,7 @@ public Void run() throws Exception {
960957
Assert.assertEquals(HttpURLConnection.HTTP_OK,
961958
conn.getResponseCode());
962959
List<String> ret = IOUtils
963-
.readLines(conn.getInputStream(), Charset.defaultCharset());
960+
.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
964961
Assert.assertEquals(1, ret.size());
965962
Assert.assertEquals("remoteuser=" + FOO_USER+ ":ugi=" + FOO_USER,
966963
ret.get(0));
@@ -969,8 +966,7 @@ public Void run() throws Exception {
969966
conn = aUrl.openConnection(url, token, OK_USER);
970967
Assert.assertEquals(HttpURLConnection.HTTP_OK,
971968
conn.getResponseCode());
972-
ret = IOUtils
973-
.readLines(conn.getInputStream(), Charset.defaultCharset());
969+
ret = IOUtils.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
974970
Assert.assertEquals(1, ret.size());
975971
Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER +
976972
":ugi=" + OK_USER, ret.get(0));
@@ -1022,8 +1018,7 @@ public Void run() throws Exception {
10221018
HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER);
10231019
Assert.assertEquals(HttpURLConnection.HTTP_OK,
10241020
conn.getResponseCode());
1025-
List<String> ret = IOUtils
1026-
.readLines(conn.getInputStream(), Charset.defaultCharset());
1021+
List<String> ret = IOUtils.readLines(conn.getInputStream(), StandardCharsets.UTF_8);
10271022
Assert.assertEquals(1, ret.size());
10281023
Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER +
10291024
":ugi=" + OK_USER, ret.get(0));

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/StatUtils.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
import java.io.BufferedReader;
2424
import java.io.InputStreamReader;
25-
import java.nio.charset.Charset;
25+
import java.nio.charset.StandardCharsets;
2626
import java.util.ArrayList;
2727
import java.util.Arrays;
2828
import java.util.List;
@@ -113,10 +113,9 @@ private static String getPermissionStringFromProcess(String[] shellCommand,
113113
ExecutorService executorService = Executors.newSingleThreadExecutor();
114114
executorService.awaitTermination(2000, TimeUnit.MILLISECONDS);
115115
try {
116-
Future<String> future =
117-
executorService.submit(() -> new BufferedReader(
118-
new InputStreamReader(process.getInputStream(),
119-
Charset.defaultCharset())).lines().findFirst().orElse(""));
116+
Future<String> future = executorService.submit(() -> new BufferedReader(
117+
new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)).lines()
118+
.findFirst().orElse(""));
120119
return future.get();
121120
} finally {
122121
process.destroy();

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import java.io.IOException;
3232
import java.io.InputStream;
3333
import java.net.URL;
34-
import java.nio.charset.Charset;
34+
import java.nio.charset.StandardCharsets;
3535
import java.util.List;
3636
import java.util.jar.JarOutputStream;
3737
import java.util.zip.ZipEntry;
@@ -135,7 +135,7 @@ public void testGetResource() throws IOException {
135135

136136
InputStream in = appClassloader.getResourceAsStream("resource.txt");
137137
assertNotNull("Resource should not be null for app classloader", in);
138-
assertEquals("hello", IOUtils.toString(in, Charset.defaultCharset()));
138+
assertEquals("hello", IOUtils.toString(in, StandardCharsets.UTF_8));
139139
}
140140

141141
private File makeTestJar() throws IOException {

‎hadoop-common-project/hadoop-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
import java.io.File;
5050
import java.io.FileNotFoundException;
5151
import java.io.IOException;
52-
import java.nio.charset.Charset;
52+
import java.nio.charset.StandardCharsets;
5353
import java.security.Principal;
5454
import java.util.HashSet;
5555
import java.util.Properties;
@@ -220,7 +220,7 @@ public static void setupKDCAndPrincipals() throws Exception {
220220
BOB_LOCALHOST, keytab_bob));
221221

222222
jaasFile = new File(kdcWorkDir, "jaas.txt");
223-
FileUtils.write(jaasFile, jaas.toString(), Charset.defaultCharset());
223+
FileUtils.write(jaasFile, jaas.toString(), StandardCharsets.UTF_8);
224224
LOG.info("\n"+ jaas);
225225
RegistrySecurity.bindJVMtoJAASFile(jaasFile);
226226
}

‎hadoop-common-project/hadoop-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import java.io.File;
2121
import java.lang.reflect.Constructor;
2222
import java.lang.reflect.Method;
23-
import java.nio.charset.Charset;
23+
import java.nio.charset.StandardCharsets;
2424
import java.security.Principal;
2525
import java.security.PrivilegedExceptionAction;
2626
import java.util.HashMap;
@@ -93,8 +93,7 @@ public void testClientLogin() throws Throwable {
9393
logLoginDetails(ALICE_LOCALHOST, client);
9494
String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
9595
assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
96-
String config = FileUtils.readFileToString(new File(confFilename),
97-
Charset.defaultCharset());
96+
String config = FileUtils.readFileToString(new File(confFilename), StandardCharsets.UTF_8);
9897
LOG.info("{}=\n{}", confFilename, config);
9998
RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
10099
} finally {
@@ -133,8 +132,7 @@ public LoginContext createLoginContextZookeeperLocalhost() throws
133132
@Test
134133
public void testKerberosAuth() throws Throwable {
135134
File krb5conf = getKdc().getKrb5conf();
136-
String krbConfig = FileUtils.readFileToString(krb5conf,
137-
Charset.defaultCharset());
135+
String krbConfig = FileUtils.readFileToString(krb5conf, StandardCharsets.UTF_8);
138136
LOG.info("krb5.conf at {}:\n{}", krb5conf, krbConfig);
139137
Subject subject = new Subject();
140138
Class<?> kerb5LoginClass =

‎hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
import java.net.HttpURLConnection;
5858
import java.net.URI;
5959
import java.net.URL;
60-
import java.nio.charset.Charset;
60+
import java.nio.charset.StandardCharsets;
6161
import java.text.MessageFormat;
6262
import java.util.ArrayList;
6363
import java.util.Arrays;
@@ -1711,8 +1711,7 @@ public void testNoRedirect() throws Exception {
17111711
conn.connect();
17121712
// Verify that we read what we wrote
17131713
Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
1714-
String content = IOUtils.toString(
1715-
conn.getInputStream(), Charset.defaultCharset());
1714+
String content = IOUtils.toString(conn.getInputStream(), StandardCharsets.UTF_8);
17161715
Assert.assertEquals(testContent, content);
17171716

17181717

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
3333

3434
import java.io.IOException;
35-
import java.nio.charset.Charset;
35+
import java.nio.charset.StandardCharsets;
3636

3737
/**
3838
* Cancels a running plan.
@@ -77,7 +77,7 @@ public void execute(CommandLine cmd) throws Exception {
7777
"Invalid plan file specified.");
7878
String planData = null;
7979
try (FSDataInputStream plan = open(planFile)) {
80-
planData = IOUtils.toString(plan, Charset.defaultCharset());
80+
planData = IOUtils.toString(plan, StandardCharsets.UTF_8);
8181
}
8282
cancelPlan(planData);
8383
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
3333

3434
import java.io.IOException;
35-
import java.nio.charset.Charset;
35+
import java.nio.charset.StandardCharsets;
3636

3737
/**
3838
* executes a given plan.
@@ -69,7 +69,7 @@ public void execute(CommandLine cmd) throws Exception {
6969

7070
String planData = null;
7171
try (FSDataInputStream plan = open(planFile)) {
72-
planData = IOUtils.toString(plan, Charset.defaultCharset());
72+
planData = IOUtils.toString(plan, StandardCharsets.UTF_8);
7373
}
7474

7575
boolean skipDateCheck = false;

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import java.io.InputStream;
2626
import java.io.StringWriter;
2727
import java.lang.reflect.Method;
28-
import java.nio.charset.Charset;
28+
import java.nio.charset.StandardCharsets;
2929

3030
import org.apache.commons.io.IOUtils;
3131
import org.apache.hadoop.conf.Configuration;
@@ -107,7 +107,7 @@ public void testDisplayForAvroFiles() throws Exception {
107107

108108
private String inputStreamToString(InputStream stream) throws IOException {
109109
StringWriter writer = new StringWriter();
110-
IOUtils.copy(stream, writer, Charset.defaultCharset());
110+
IOUtils.copy(stream, writer, StandardCharsets.UTF_8);
111111
return writer.toString();
112112
}
113113

0 commit comments

Comments
 (0)