Install CDH5.11 on CentOS 7

##1)Environmentinitialization
Note：Atleastthreenodes,theMasternodeatleast6Gofmemory,andtheothertwoatleast4Gofmemory

|--------------|-----------|-------------|----------------|-----------|--------------------|-------------|
|Host|Server|Agent|zookeeper|namenode|secondarynamenode|datanode|
|--------------|-----------|-------------|----------------|-----------|--------------------|-------------|	
|cdh01Master|Y|Y|Y|Y|||
|--------------|-----------|-------------|----------------|-----------|--------------------|-------------|
|cdh02slave||Y|Y||Y|Y|
|--------------|-----------|-------------|----------------|-----------|--------------------|-------------|
|cdh03slave||Y|Y|||Y|
|--------------|-----------|-------------|----------------|-----------|--------------------|-------------|
|cdh04slave||Y|Y|||Y|
|--------------|-----------|-------------|----------------|-----------|--------------------|-------------|
|cdh05slave||Y|Y|||Y|
|______________|___________|_____________|________________|___________|____________________|_____________|


#######################################
##Nameresolutiononeverynodes
########################
cat>/etc/hosts<<EOF
127.0.0.1localhost
192.168.51.6cdh01
192.168.51.7cdh02
192.168.51.16cdh03
192.168.51.17cdh04
192.168.51.4cdh05
EOF

##Wemusttodisableipv6oneverynodes
echo'net.ipv6.conf.all.disable_ipv6=1'>>/etc/sysctl.conf
echo'net.ipv6.conf.default.disable_ipv6=1'>>/etc/sysctl.conf
echo'vm.swappiness=0'>>/etc/sysctl.conf
sysctl-p
echonever>/sys/kernel/mm/transparent_hugepage/defrag
echo'echonever>/sys/kernel/mm/transparent_hugepage/defrag'>>/etc/rc.local
chmod+x/etc/rc.d/rc.local

groupaddhadoop-g700
useraddhadoop-ghadoop-u700
echo"hadoop123"|passwd--stdinhadoop
echo'hadoopALL=(ALL)NOPASSWD:ALL'>>/etc/sudoers

echo"ulimit-SHn204800">>/etc/rc.local
echo"ulimit-SHu204800">>/etc/rc.local
cat>>/etc/security/limits.conf<<EOF
*softnofile204800
*hardnofile204800
*softnproc204800
*hardnproc204800
EOF

#Installjdkoneverynodes
wgethttp://god.nongdingbang.net/downloads/auto_jdk_1.8.sh
shauto_jdk_1.8.sh

#Installjava-jdbconeverynodes
cd/home/tools
wgethttp://dev.MysqL.com/get/Downloads/Connector-J/MysqL-connector-java-5.1.39.tar.gz
tarzxvfMysqL-connector-java-5.1.39.tar.gz
mkdir/usr/share/java/
\cpMysqL-connector-java-5.1.39/MysqL-connector-java-5.1.39-bin.jar/usr/share/java/MysqL-connector-java.jar
########################################################################################################

##2)DeploylocalyumforCDHoncdh01
yum-yinstallepel-release
yum-yinstallNginxcreaterepo

cat>/etc/Nginx/Nginx.conf<<EOF
userNginx;
worker_processesauto;
error_log/var/log/Nginx/error.log;
pid/run/Nginx.pid;
#Loaddynamicmodules.See/usr/share/Nginx/README.dynamic.
include/usr/share/Nginx/modules/*.conf;
events{
worker_connections102400;
}
http{
log_formatmain'$remote_addr-$remote_user[$time_local]"$request"'
'$status$body_bytes_sent"$http_referer"'
'"$http_user_agent""$http_x_forwarded_for"';
access_log/var/log/Nginx/access.logmain;
sendfileon;
tcp_nopushon;
tcp_nodelayon;
keepalive_timeout120;
types_hash_max_size2048;
client_max_body_size1G;
client_header_timeout10;
client_body_timeout10;
fastcgi_connect_timeout300;
fastcgi_send_timeout300;
fastcgi_read_timeout300;
fastcgi_buffer_size64k;
fastcgi_buffers464k;
fastcgi_busy_buffers_size128k;
fastcgi_temp_file_write_size128k;
server_names_hash_bucket_size64;
gzipon;
gzip_min_length1024;
gzip_buffers464k;
gzip_http_version1.1;
gzip_comp_level9;
gzip_typestext/plaintext/csstext/xmltext/x-componenttext/javascriptapplication/x-javascriptapplication/javascriptapplication/jsonapplication/xhtml+xmlimage/svg+xmlimage/x-iconimage/gifimage/pngimage/jpegvideo/mp4video/x-flvvideo/x-m4v;
gzip_disable"msie6";
gzip_varyon;
include/etc/Nginx/mime.types;
default_typeapplication/octet-stream;
include/etc/Nginx/conf.d/*.conf;
}
EOF

mkdir-p/data/www/html/

cat>/etc/Nginx/conf.d/default.conf<<EOF
server{
listen80;
server_namelocalhost;
#charsetkoi8-r;
#access_log/var/log/Nginx/log/host.access.logmain;
location/{
root/data/www/html;
autoindexon;#允许浏览文件
}
#error_page404/404.html;
#redirectservererrorpagestothestaticpage/50x.html
#
error_page500502503504/50x.html;
location=/50x.html{
root/usr/share/Nginx/html;
}
}
EOF

cd/data/www/html/&&wgethttp://god.nongdingbang.net/downloads/cm5.11.0-centos7.tar.gz
tarzxvfcm5.11.0-centos7.tar.gz&&chown-RNginx.Nginx/data/www/html
cd/data/www/html/cm/5.11&&createrepo.

#runthiscommandoneverynodes
cat>/etc/yum.repos.d/cloudera-cm.repo<<EOF
[cloudera-cm]
#PackagesforCloudera'sDistributionforcm,Version5,onRedHatorCentOS7x86_64
name=Cloudera'sDistributionforcm,Version5
baseurl=http://192.168.51.6/cm/5.11
gpgcheck=0
EOF

#StartNginxoncdh01
serviceNginxstart&&chkconfigNginxon


##3)Installcloudera-agentoneverynodes
yuminstall-yvimwgetopenssh-clientspythonchronycloudera-manager-agentcloudera-manager-daemons


##4)Installcloudera-mageneroncdh01
yuminstall-ycloudera-manager-daemonscloudera-manager-server


##5)InstallMysqLServeroncdh01
cd/home/tools
yum-yinstallnumactllibaionet-tools
wgethttp://god.nongdingbang.net/downloads/MysqL-5.7-el7.tgz&&tarzxvfMysqL-5.7-el7.tgz
if[$?-eq0];then
rpm-UvhMysqL*.rpm
fi
mkdir-p/data/{MysqL_data,MysqL_log,MysqL_slow,MysqL_undo}
chown-RMysqL.MysqL/data/*
cat>/etc/my.cnf<<EOF
[client]
port=3306
socket=/tmp/MysqL.sock
[MysqL]
no-auto-rehash
[MysqLd]
port=3306
character-set-server=utf8
socket=/tmp/MysqL.sock
datadir=/data/MysqL_data
explicit_defaults_for_timestamp=true
lower_case_table_names=1
sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'
back_log=103
max_connections=3000
max_connect_errors=100000
table_open_cache=512
external-locking=FALSE
max_allowed_packet=32M
sort_buffer_size=2M
join_buffer_size=2M
thread_cache_size=51
query_cache_size=0
query_cache_type=0
transaction_isolation=REPEATABLE-READ
tmp_table_size=96M
max_heap_table_size=96M
log-error=/data/MysqL_log/error.log
###***slowqueryparameters
long_query_time=2
slow_query_log=1
slow_query_log_file=/data/MysqL_slow/slow.log
###***binlogparameters
log-bin=MysqL-bin
binlog_cache_size=4M
max_binlog_cache_size=4096M
max_binlog_size=1024M
binlog_format=ROW
binlog_row_image=minimal
expire_logs_days=3
sync_binlog=0
###***undolog
innodb_undo_directory=/data/MysqL_undo
innodb_undo_logs=128
innodb_undo_tablespaces=4
innodb_undo_log_truncate=1
innodb_max_undo_log_size=1G
innodb_purge_rseg_truncate_frequency
#***MyISAMparameters
key_buffer_size=16M
read_buffer_size=1M
read_rnd_buffer_size=16M
bulk_insert_buffer_size=1M
###***master-slavereplicationparameters
server-id=1
#slave-skip-errors=all
#***Innodbstorageengineparameters
innodb_buffer_pool_dump_at_shutdown=1
innodb_buffer_pool_load_at_startup=1
innodb_buffer_pool_size=16G
innodb_data_file_path=ibdata1:10M:autoextend
innodb_temp_data_file_path=ibtmp1:12M:autoextend:max:5G
#innodb_file_io_threads=8
innodb_thread_concurrency=0
innodb_flush_log_at_trx_commit=2
innodb_log_buffer_size=16M
innodb_log_file_size=512M
innodb_log_files_in_group=2
innodb_max_dirty_pages_pct=75
innodb_buffer_pool_dump_pct=50
innodb_lock_wait_timeout=50
innodb_file_per_table=on
innodb_flush_neighbors=1
innodb_io_capacity_max=2000
wait_timeout=14400
interactive_timeout=14400
[MysqLdump]
quick
max_allowed_packet=32M

[myisamchk]
key_buffer=16M
sort_buffer_size=16M
read_buffer=8M
write_buffer=8M

[MysqLd_safe]
open-files-limit=28192
log-error=/data/MysqL_log/error.log
pid-file=/data/MysqL_data/MysqLd.pid
EOF


##6)InitializeMysqLconfiguration
MysqLd--defaults-file=/etc/my.cnf--user=MysqL\
--datadir=/data/MysqL_data--initialize-insecure


##7)StartMysqL
systemctlrestartMysqLd&&systemctlenableMysqLd


##8)settingroot'spasswordforMysqLoncdh01
MysqL-e"ALTERUSER'root'@'localhost'IDENTIFIEDBY'Ad@sd119';"
MysqL-uroot-pAd@sd119-DMysqL-e"selectuser,host,authentication_string,password_expiredfromuser;"

MysqL-uroot-pAd@sd119-e"createdatabasescmdefaultcharactersetutf8;grantallprivilegeson*.*toscm@'%'identifiedby'scm';"
MysqL-uroot-pAd@sd119-e"createdatabaseamondefaultcharactersetutf8;grantallprivilegesonamon.*to'amon'@'%'identifiedby'amon_password';"
MysqL-uroot-pAd@sd119-e"createdatabasermandefaultcharactersetutf8;grantallprivilegesonrman.*to'rman'@'%'identifiedby'rman_password';"
MysqL-uroot-pAd@sd119-e"createdatabaseMetastoredefaultcharactersetutf8;grantallprivilegesonMetastore.*to'hive'@'%'identifiedby'hive_password'"
MysqL-uroot-pAd@sd119-e"createdatabasesentrydefaultcharactersetutf8;grantallprivilegesonsentry.*to'sentry'@'%'identifiedby'sentry_password'"
MysqL-uroot-pAd@sd119-e"createdatabasenavdefaultcharactersetutf8;grantallprivilegesonnav.*to'nav'@'%'identifiedby'nav_password'"
MysqL-uroot-pAd@sd119-e"createdatabasenavmsdefaultcharactersetutf8;grantallprivilegesonnavms.*to'navms'@'%'identifiedby'navms_password';"
MysqL-uroot-pAd@sd119-e"createdatabasehuedefaultcharactersetutf8;grantallprivilegesonhue.*to'hue'@'%'identifiedby'hue_password'"
MysqL-uroot-pAd@sd119-e"flushprivileges;"

#9)Createthisdirectoryoneverynodes
mkdir-p/opt/cloudera/parcels
chowncloudera-scm:cloudera-scm/opt/cloudera/parcels


#10)StartServeroncdh01
#############################
cat>/etc/cloudera-scm-server/db.properties<<EOF
com.cloudera.cmf.db.type=MysqL
com.cloudera.cmf.db.host=cdh01
com.cloudera.cmf.db.name=scm
com.cloudera.cmf.db.user=scm
com.cloudera.cmf.db.password=scm
com.cloudera.cmf.db.setupType=EXTERNAL
EOF

cd/opt/cloudera/parcel-repo

wget-chttp://god.nongdingbang.net/downloads/parcel-repo.tgz
if[$?-eq0];then
tarzxvfparcel-repo.tgz
fi

mvparcel-repo/*.
rm-rfparcel-repo*

/etc/init.d/cloudera-scm-serverstart
chkconfigcloudera-scm-serveron


#11)Startagentdoneverynodes
sed-i's/server_host=localhost/server_host=192.168.51.6/'/etc/cloudera-scm-agent/config.ini
sed-i's#\#parcel_dir=/opt/cloudera/parcels#parcel_dir=/opt/cloudera/parcels#'/etc/cloudera-scm-agent/config.ini

cat>/etc/cloudera-scm-agent/config.ini<<EOF
server_host=cdh01
server_port=7182
parcel_dir=/opt/cloudera/parcels
EOF

/etc/init.d/cloudera-scm-agentstart
chkconfigcloudera-scm-agenton

##12)Browseyourcdh
http://192.168.51.6:7180/cmf/login
Install CDH5.11 on CentOS 7

猜你在找的CentOS相关文章
