🔰 11.1 Configure Hadoop and start cluster
services using Ansible Playbook .
NAMENODE:
setup : Running Virtual Box on top of Windows.
we have 4 virtual box
- for Ansible
- for Name Node
- for Data Node
- for Client
On the left we have ansible system and right we have fresh VM with rhel-8 os .
It does not have jdk and hadoop running , we have to build Name Node here from scratch .
After playbook execute successfully NameNode will start .
Done , NameNode configured successfully
We can also see from WEB_UI through url………http://localhost:50070
DATANODE:
Running data-node playbook
After running playbook , datanode configured for hadoop master and joined to the hdfs cluster .
We can see from the WebUI also that some storage is contributed by the slave or data node .
Now lets configure client in the hdfs cluster ,
for putting file from client to hdfs cluster we use hadoop fs -put file_name /
As we can see from client we put the file anuj.txt and it come up in CLUSTER .
So , we successfully created Hadoop HDFS cluster , and also uploaded one file in cluster.
Entire playbook for NameNode :*******************************
— -
- hosts: 192.168.43.31
vars:
— NNdir: “Name_Node_Dir_by_Ansible”
tasks:
— name: making directory for Name Node
file:
path: /{{ NNdir }}
state: directory
- name: Stop service firewalld, if started
service:
name: firewalld
state: stopped
- name: Disable SELinux
selinux:
state: disabled
- name: transferring softwear via n/w
copy:
src: “/root/jdk-8u171-linux-x64.rpm”
dest: “/root/”
- name: transferring hadoop softwear
copy:
src: “/root/hadoop-1.2.1–1.x86_64.rpm”
dest: “/root/”
- name: installing jdk softwear
shell: rpm -ivh jdk-8u171-linux-x64.rpm
ignore_errors: yes
- name: installing hadoop softwear
shell: rpm -ivh hadoop-1.2.1–1.x86_64.rpm — force
ignore_errors: yes
- name: copying corefile-html
copy:
src: “/root/arth-ws/ansible-ws/hadoop/conf_file/NN_core-site_template.xml”
dest: “/etc/hadoop/core-site.xml”
- name: copying dynamic data in hdfs file template and sending .
template:
src: “/root/arth-ws/ansible-ws/hadoop/conf_file/NN_hdfs-site_template.xml”
dest: “/etc/hadoop/hdfs-site.xml”
- name: formatting namenode
shell: “echo Y | hadoop namenode -format”
ignore_errors: yes
- name: starting namenode
shell: “hadoop-daemon.sh start namenode”
ignore_errors: yes
- name: “checking through dfsadmin report”
shell: hadoop dfsadmin -report
register: op_report
- name: “displaying report”
debug:
var: op_report
DATANODE-playbook**************************************
— -
- hosts: 192.168.43.205
vars:
— DNdir: “Data_Node_Dir_by_Ansible”
— NNip: 192.168.43.31
tasks:
— name: making directory for Data Node
file:
path: /{{ DNdir }}
state: directory
- name: Stop service firewalld, if started
service:
name: firewalld
state: stopped
- name: Disable SELinux
selinux:
state: disabled
- name: transferring softwear via n/w
copy:
src: “/root/jdk-8u171-linux-x64.rpm”
dest: “/root/”
- name: transferring hadoop softwear
copy:
src: “/root/hadoop-1.2.1–1.x86_64.rpm”
dest: “/root/”
- name: installing jdk softwear
shell: rpm -ivh jdk-8u171-linux-x64.rpm
ignore_errors: yes
- name: installing hadoop softwear
shell: rpm -ivh hadoop-1.2.1–1.x86_64.rpm — force
ignore_errors: yes
- name: copying corefile-html
copy:
src: “/root/arth-ws/ansible-ws/hadoop/conf_file/DN_core-site_template.xml”
dest: “/etc/hadoop/core-site.xml”
- name: copying dynamic data in hdfs file template and sending .
template:
src: “/root/arth-ws/ansible-ws/hadoop/conf_file/DN_hdfs-site_template.xml”
dest: “/etc/hadoop/hdfs-site.xml”
- name: starting data node
shell: hadoop-daemon.sh start datanode
- shell: hadoop dfsadmin -report
register: report
— debug:
var: report
Client-playbook:********************************************
— -
- hosts: 192.168.43.37
tasks:
- name: Stop service firewalld, if started
service:
name: firewalld
state: stopped
- name: Disable SELinux
selinux:
state: disabled
- name: transferring softwear via n/w
copy:
src: “/root/jdk-8u171-linux-x64.rpm”
dest: “/root/”
- name: transferring hadoop softwear
copy:
src: “/root/hadoop-1.2.1–1.x86_64.rpm”
dest: “/root/”
- name: installing jdk softwear
shell: rpm -ivh jdk-8u171-linux-x64.rpm
ignore_errors: yes
- name: installing hadoop softwear
shell: rpm -ivh hadoop-1.2.1–1.x86_64.rpm — force
ignore_errors: yes
- name: copying corefile-html
copy:
src: “/root/arth-ws/ansible-ws/hadoop/conf_file/DN_core-site_template.xml”
dest: “/etc/hadoop/core-site.xml”
- name: “checking through dfsadmin report”
shell: hadoop dfsadmin -report
register: op_report
- name: “displaying report”
debug:
var: op_report
