Skip to content

Commit 092cc6e

Browse files
jeevbidjeevbid
committed
update to latest configs; install sudo
1 parent 1f56d9a commit 092cc6e

15 files changed

+334
-95
lines changed

‎spark-jupyter/Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ RUN apt-get update -y \
3434
&& apt-get upgrade -y \
3535
&& apt-get -y install openjdk-8-jdk wget openssh-server sshpass supervisor \
3636
&& apt-get -y install nano net-tools lynx \
37+
&& apt-get -y install sudo \
3738
&& apt-get clean
3839

3940
# setup ssh

‎spark-jupyter/build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
docker build --no-cache -t spark-jupyter:local .
3+
docker build --no-cache --platform=linux/amd64 -t spark-jupyter:local .

‎spark-jupyter/run.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
docker run -it \
4+
-p 9870:9870 \
5+
-p 8088:8088 \
6+
-p 8080:8080 \
7+
-p 18080:18080 \
8+
-p 9000:9000 \
9+
-p 8888:8888 \
10+
-p 9864:9864 \
11+
-e PYSPARK_MASTER=spark://localhost:7077 \
12+
spark-jupyter:local

‎spark-jupyter/ubuntu/usr/local/hadoop/etc/hadoop/capacity-scheduler.xml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,4 +217,28 @@
217217
</description>
218218
</property>
219219

220+
<property>
221+
<name>yarn.scheduler.capacity.workflow-priority-mappings</name>
222+
<value></value>
223+
<description>
224+
A list of mappings that will be used to override application priority.
225+
The syntax for this list is
226+
[workflowId]:[full_queue_name]:[priority][,next mapping]*
227+
where an application submitted (or mapped to) queue "full_queue_name"
228+
and workflowId "workflowId" (as specified in application submission
229+
context) will be given priority "priority".
230+
</description>
231+
</property>
232+
233+
<property>
234+
<name>yarn.scheduler.capacity.workflow-priority-mappings-override.enable</name>
235+
<value>false</value>
236+
<description>
237+
If a priority mapping is present, will it override the value specified
238+
by the user? This can be used by administrators to give applications a
239+
priority that is different than the one specified by the user.
240+
The default is false.
241+
</description>
242+
</property>
243+
220244
</configuration>

‎spark-jupyter/ubuntu/usr/local/hadoop/etc/hadoop/container-executor.cfg

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,27 @@ feature.tc.enabled=false
88
#[docker]
99
# module.enabled=## enable/disable the module. set to "true" to enable, disabled by default
1010
# docker.binary=/usr/bin/docker
11-
# docker.allowed.capabilities=## comma seperated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
12-
# docker.allowed.devices=## comma seperated list of devices that can be mounted into a container
13-
# docker.allowed.networks=## comma seperated networks that can be used. e.g bridge,host,none
14-
# docker.allowed.ro-mounts=## comma seperated volumes that can be mounted as read-only
15-
# docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
11+
# docker.allowed.capabilities=## comma separated capabilities that can be granted, e.g CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
12+
# docker.allowed.devices=## comma separated list of devices that can be mounted into a container
13+
# docker.allowed.networks=## comma separated networks that can be used. e.g bridge,host,none
14+
# docker.allowed.ro-mounts=## comma separated volumes that can be mounted as read-only
15+
# docker.allowed.rw-mounts=## comma separate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
1616
# docker.privileged-containers.enabled=false
17-
# docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers
17+
# docker.allowed.volume-drivers=## comma separated list of allowed volume-drivers
1818
# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default
19-
# docker.allowed.runtimes=## comma seperated runtimes that can be used.
19+
# docker.allowed.runtimes=## comma separated runtimes that can be used.
2020

2121
# The configs below deal with settings for FPGA resource
2222
#[fpga]
2323
# module.enabled=## Enable/Disable the FPGA resource handler module. set to "true" to enable, disabled by default
2424
# fpga.major-device-number=## Major device number of FPGA, by default is 246. Strongly recommend setting this
25-
# fpga.allowed-device-minor-numbers=## Comma separated allowed minor device numbers, empty means all FPGA devices managed by YARN.
25+
# fpga.allowed-device-minor-numbers=## Comma separated allowed minor device numbers, empty means all FPGA devices managed by YARN.
26+
27+
# The configs below deal with settings for resource handled by pluggable device plugin framework
28+
#[devices]
29+
# module.enabled=## Enable/Disable the device resource handler module for isolation. Disabled by default.
30+
# devices.denied-numbers=## Blacklisted devices not permitted to use. The format is comma separated "majorNumber:minorNumber". For instance, "195:1,195:2". Leave it empty means default devices reported by device plugin are all allowed.
31+
32+
# The configs below deal with settings for GPU resource
33+
#[gpu]
34+
# module.enabled=## Enable/Disable GPU resource handler module. Set to "true" to enable, disabled by default

‎spark-jupyter/ubuntu/usr/local/hadoop/etc/hadoop/hadoop-env.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@
5353
# variable is REQUIRED on ALL platforms except OS X!
5454
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
5555

56+
# The language environment in which Hadoop runs. Use the English
57+
# environment to ensure that logs are printed as expected.
58+
export LANG=en_US.UTF-8
59+
5660
# Location of Hadoop. By default, Hadoop will attempt to determine
5761
# this location based upon its execution path.
5862
export HADOOP_HOME=/usr/local/hadoop

‎spark-jupyter/ubuntu/usr/local/hadoop/etc/hadoop/hadoop-policy.xml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,16 @@
6363
A special value of "*" means all users are allowed.</description>
6464
</property>
6565

66+
<property>
67+
<name>security.datanode.lifeline.protocol.acl</name>
68+
<value>*</value>
69+
<description>ACL for DatanodeLifelineProtocol, which is used by a
70+
DataNode to send lifeline messages to the NameNode.
71+
The ACL is a comma-separated list of user and group names. The user and
72+
group list is separated by a blank. For e.g. "alice,bob users,wheel".
73+
A special value of "*" means all users are allowed.</description>
74+
</property>
75+
6676
<property>
6777
<name>security.namenode.protocol.acl</name>
6878
<value>*</value>
@@ -82,6 +92,16 @@
8292
A special value of "*" means all users are allowed.</description>
8393
</property>
8494

95+
<property>
96+
<name>security.get.user.mappings.protocol.acl</name>
97+
<value>*</value>
98+
<description>ACL for GetUserMappingsProtocol, implemented by the NameNode
99+
and Job Tracker which maps users to groups.
100+
The ACL is a comma-separated list of user and group names. The user and
101+
group list is separated by a blank. For e.g. "alice,bob users,wheel".
102+
A special value of "*" means all users are allowed.</description>
103+
</property>
104+
85105
<property>
86106
<name>security.refresh.user.mappings.protocol.acl</name>
87107
<value>*</value>
@@ -92,6 +112,16 @@
92112
users are allowed.</description>
93113
</property>
94114

115+
<property>
116+
<name>security.reconfiguration.protocol.acl</name>
117+
<value>*</value>
118+
<description>ACL for ReconfigurationProtocol, used by HDFS admin to
119+
reload configuration for NameNode/DataNode without restarting them.
120+
The ACL is a comma-separated list of user and group names. The user and
121+
group list is separated by a blank. For e.g. "alice,bob users,wheel".
122+
A special value of "*" means all users are allowed.</description>
123+
</property>
124+
95125
<property>
96126
<name>security.refresh.policy.protocol.acl</name>
97127
<value>*</value>
@@ -102,13 +132,43 @@
102132
A special value of "*" means all users are allowed.</description>
103133
</property>
104134

135+
<property>
136+
<name>security.refresh.callqueue.protocol.acl</name>
137+
<value>*</value>
138+
<description>ACL for RefreshCallQueueProtocol, which is used to refresh
139+
the call queue in use currently.
140+
The ACL is a comma-separated list of user and group names. The user and
141+
group list is separated by a blank. For e.g. "alice,bob users,wheel".
142+
A special value of "*" means all users are allowed.</description>
143+
</property>
144+
145+
<property>
146+
<name>security.refresh.generic.protocol.acl</name>
147+
<value>*</value>
148+
<description>ACL for GenericRefreshProtocol, which is used to refresh
149+
arbitrary things at runtime.
150+
The ACL is a comma-separated list of user and group names. The user and
151+
group list is separated by a blank. For e.g. "alice,bob users,wheel".
152+
A special value of "*" means all users are allowed.</description>
153+
</property>
154+
105155
<property>
106156
<name>security.ha.service.protocol.acl</name>
107157
<value>*</value>
108158
<description>ACL for HAService protocol used by HAAdmin to manage the
109159
active and stand-by states of namenode.</description>
110160
</property>
111161

162+
<property>
163+
<name>security.router.admin.protocol.acl</name>
164+
<value>*</value>
165+
<description>ACL for RouterAdmin Protocol. The ACL is a comma-separated
166+
list of user and group names. The user and
167+
group list is separated by a blank. For e.g. "alice,bob users,wheel".
168+
A special value of "*" means all users are allowed.
169+
</description>
170+
</property>
171+
112172
<property>
113173
<name>security.zkfc.protocol.acl</name>
114174
<value>*</value>

‎spark-jupyter/ubuntu/usr/local/hadoop/etc/hadoop/hdfs-rbf-site.xml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
https://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
-->
15+
16+
<!-- Put site-specific property overrides in this file. -->
17+
18+
<configuration>
19+
20+
</configuration>

‎spark-jupyter/ubuntu/usr/local/hadoop/etc/hadoop/httpfs-signature.secret

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)