Apache Hadoop Cluster Construction Process

vi /etc/sysconfig/network-scripts/ifcfg-eth0

BOOTPROTO=static
IPADDR=192.168.31.210
NETMASK=255.255.255.0
GATEWAY=192.168.31.1
DNS1=8.8.8.8
DNS2=114.114.114.114
ONBOOT=yes

 

service network restart

ping http://www.baidu.com

ip addr

yum install vim -y

vim /etc/hostname

hadoop01

hostnamectl set-hostname hadoop01

hostname -f

vim /etc/hosts

192.168.31.201 hadoop01
192.168.31.202 hadoop02
192.168.31.203 hadoop03

 

ssh-keygen -t rsa

ssh-copy-id hadoop01

vim authorized_keys

chmod 700 .ssh

chmod 600 .ssh/*

vim /etc/ssh/sshd_config

UseDNS no

systemctl restart sshd

systemctl status sshd

cat ~/.ssh/id_rsa.pub | ssh hadoop02 ‘cat – >> ~/.ssh/authorized_keys’

cat ~/.ssh/id_rsa.pub | ssh hadoop03 ‘cat – >> ~/.ssh/authorized_keys’

scp /etc/hosts hdp01:/etc/

scp /etc/hosts hdp02:/etc/

curl https://pypi.python.org/packages/source/p/pssh/pssh-2.3.1.tar.gz

tar xvf pssh-2.3.1.tar.gz

python setup.py install

pssh –version

pssh -H root@hadoop02

date

vim cdhs

root@cdh00
root@cdh01
root@cdh02

 

systemctl stop firewalld

systemctl disable firewalld

systemctl status firewalld

vim /etc/selinux/config

SELINUX=disabled

sed -i ‘s/^SELINUX=enforcing$/SELINUX=disabled/’ /etc/selinux/config

create external table sogoulogs (id string,datatime string,userid string,searchname string,relorder string,cliorder string,cliurl string) STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' WITH SERDEPROPERTIES("hbase.columns.mapping"=":key,info:datatime,info:userid,info,searchname,info:relorder,info:cliorder,info:cliurl") TBLPROPERTIES("hbase.table.name"="sogoulogs");

 

val line = sc.textFile line.flatMap(.split(" ")).map((,1)).reduceByKey(+).collect().foreach(println)

# zookeeper (all) zookeeper/bin/zkServersh start

# jps: QuorumPeerMain

# localhost:2181

# journalnode (all) hadoop/sbin/hadoop-daemon.sh start journalnode

# namenode hadoop/bin/hdfs namenode -format hadoop/bin/hdfs zkfc -formatZK hadoop/bin/hdfs namenode #hadoop01 hadoop/bin/hdfs namenode -bootstrapStandby #hadoop02 hadoop/sbin/hadoop-daemon.sh stop journalnode #(all) # hdfs hadoop/sbin/start-dfs.sh

# jps: DFSZKFailoverController, journalnode, namenode, datanode

# localhost:50070

# yarn hadoop/sbin/start-yarn.sh hadoop/sbin/yarn-daemon.sh start resourcemanager hadoop/bin/yarn –daemon start resourcemanager

#上一条命令不能用的情况下

# jps: nodemanager(hadoop01) resourcemanager(hadoop02)

# localhost:8088 # hbase hbase/bin/start-hbase.sh

# jps: HMaster HRegionServer # localhost:16010

# kafka(all) kafka/bin/kafka-server-start.sh config/*.properties

# jps:kafka

# flume(all) flume/bin/flume-ng agent -n agent_name -c conf -f flume/conf/flume-conf.properties -Dfllume.root.logger=INFO,console

# hive hive/bin/hive

# spark spark/bin/spark-shell

# flink flink/bin/start-cluster.sh

# localhost:8081

hadoop3.x: HDFS NameNode

内部通讯接口：8020/9000/9820 HDFS NameNode

对用户的查询接口：web 端 9870（服务端口号 9820）

Yarn 查看任务运行情况：

web 端 8088（服务端口号 8032）

历史服务器： web 端 19888(不用直接进入，点击 history 选项进入）（服务端口号 10020）

hadoop2.x: HDFS NameNode 内部通讯接口：8020/9000 HDFS NameNode 对用户的查询接口：50070 Yarn 查看任务运行情况：8088

历史服务器 19888

2181：zookeeper 的端口号：

2888：单个服务器与集群 Leader 服务器交换信息的端口号

3888：执行选举时服务器相互通信的端口号

6379：Redis 的端口号

16010：HBASE 的 web 端口号

16000: HBASE 的服务端口号

8485: journalnode 默认的端口号

9092：kafka 的端口号

41414：flume 监控的端口