搭建Hadoop高可用集群

搭建Hadoop高可用集群

1. JDK安装

1
2
3
4
5
6
7
8
tar -zxvf jdk-8u161-linux-x64.tar.gz -C /export/servers

mv jdk1.8.0_161/ jdk

# 配置jdk系统环境变量
export JAVA_HOME=/export/servers/jdk
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

2. Hadoop安装

1
2
3
4
5
tar -zxvf hadoop2.7.4.tar.gz -C /export/servers

# 配置环境变量
export HADOOP_HOME=/export/servers/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

3. zookeeper安装

1
2
3
4
5
6
7
tar -zxvf zookeeper-3.4.10.tar.gz -C /export/servers

cd /export/servers/zookeeper/conf

cp zoo_sample.cfg zoo.cfg

vi zoo.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=//export/data/zookeeper/zkdata
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
server.1=node-01:2888:3888
server.2=node-02:2888:3888
server.3=node-03:2888:3888
1
2
3
4
5
6
7
# 创建myid文件
mkdir -p /export/data/zookeeper/zkdata

cd /export/data/zookeeper/zkdata

echo 1 > myid
### 注意：是第几台服务器就写入几
1
2
3
# 配置环境变量
export ZK_HOME=/export/servers/zookeeper
export PATH=$PATH:JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin
1
2
3
4
scp -r /export/ node-02:/
scp -r /export/ node-03:/
scp /etc/profile node-02:/etc/profile
scp /etc/profile node-03:/etc/profile

4. 文件配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
	<property>
		<name>yarn.nodemanager.resource.memory-mb</name>
		<value>2048</value>
	</property>
	<property>
		<name>yarn.scheduler.maximum-allocation-mb</name>
		<value>2048</value>
	</property>
	<property>
		<name>yarn.nodemanager.resource.cpu-vcores</name>
		<value>1</value>
	</property>
	<property>
                <name>yarn.resourcemanager.ha.enabled</name>
                <value>true</value>
        </property>
	<property>
                <name>yarn.resourcemanager.cluster-id</name>
                <value>yrc</value>
        </property>
	<property>
                <name>yarn.resourcemanager.ha.rm-ids</name>
                <value>rm1,rm2</value>
        </property>
	<property>
                <name>yarn.resourcemanager.hostname.rm1</name>
                <value>node-01</value>
        </property>
	<property>
                <name>yarn.resourcemanager.hostname.rm2</name>
                <value>node-02</value>
        </property>
	<property>
                <name>yarn.resourcemanager.zk-address</name>
                <value>node-01:2181,node-02:2181,node-03:2181</value>
        </property>
	<property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
</configuration>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
</configuration>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
		<name>dfs.replication</name>
		<value>2</value>
	</property>
	<property>
		<name>dfs.namenode.name.dir</name>
		<value>file:/export/data/hadoop/name</value>
	</property>
	<property>
                <name>dfs.datanode.data.dir</name>
                <value>file:/export/data/hadoop/data</value>
        </property>
	<property>
                <name>dfs.webhdfs.enabled</name>
                <value>true</value>
        </property>
	<property>
                <name>dfs.nameservices</name>
                <value>ns1</value>
        </property>
	<property>
                <name>dfs.ha.namenodes.ns1</name>
                <value>nn1,nn2</value>
        </property>
	<property>
                <name>dfs.namenode.rpc-address.ns1.nn1</name>
                <value>node-01:9000</value>
        </property>
	<property>
                <name>dfs.namenode.http-address.ns1.nn1</name>
                <value>node-01:50070</value>
        </property>
	<property>
                <name>dfs.namenode.rpc-address.ns1.nn2</name>
                <value>node-02:9000</value>
        </property>
	<property>
                <name>dfs.namenode.http-address.ns1.nn2</name>
                <value>node-02:50070</value>
        </property>
	<property>
                <name>dfs.namenode.shared.edits.dir</name>
                <value>qjournal://node-01:8485;node-02:8485;node-03:8485/ns1</value>
        </property>
	<property>
                <name>dfs.journalnode.edits.dir</name>
                <value>/export/data/hadoop/journaldata</value>
        </property>
	<property>
                <name>dfs.ha.automatic-failover.enabled</name>
                <value>true</value>
        </property>
	<property>
                <name>dfs.client.failover.proxy.provider.ns1</name>
                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
	<property>
                <name>dfs.ha.fencing.methods</name>
                <value>
					sshfence
					shell(/bin/true)
				</value>
        </property>
	<property>
                <name>dfs.ha.fencing.ssh.private-key-files</name>
                <value>/root/.ssh/id_rsa</value>
        </property>
	<property>
                <name>dfs.ha.fencing.ssh.connect-timeout</name>
                <value>30000</value>
        </property>
</configuration>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://ns1</value>
	</property>
	<property>
		<name>hadoop.tmp.dir</name>
		<value>/export/servers/hadoop/tmp</value>
	</property>
	<property>
		<name>ha.zookeeper.quorum</name>
		<value>node-01:2181,node-02:2181,node-03:2181</value>
	</property>
</configuration>

5. 启动

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# 各个节点
zkServer.sh start

# 各个节点
hadoop-daemon.sh start journalnode

# node-01节点
hadoop namenode -format
scp -r /export/data/hadoop node-02:/export/data/

# node-01节点
hdfs zkfc -formatZK

start-dfs.sh

start-yarn.sh