{"id":5184,"date":"2022-04-12T13:49:41","date_gmt":"2022-04-12T04:49:41","guid":{"rendered":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=5184"},"modified":"2022-04-19T13:43:23","modified_gmt":"2022-04-19T04:43:23","slug":"hadoop-cluster-%ea%b5%ac%ec%84%b1%ed%95%98%ea%b8%b0","status":"publish","type":"post","link":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=5184","title":{"rendered":"Hadoop Cluster \uad6c\uc131\ud558\uae30(with Spark)"},"content":{"rendered":"

Hadoop Cluster \uad6c\uc131\ud558\uae30(with Spark)<\/h1>\n

\ucc38\uc870 : AWS EC2 \ub97c \uc774\uc6a9\ud574 \uad6c\uc131<\/a><\/p>\n

\ucc38\uc870 : \uac00\uc0c1\uba38\uc2e0\uc744 \uc774\uc6a9\ud574 \uad6c\uc131<\/a><\/p>\n

\uae30\ubcf8 \ubc29\ud5a5<\/h2>\n
    \n
  1. \n

    EC2 \uc778\uc2a4\ud134\uc2a4 \ud558\ub098 \uc0dd\uc131<\/p>\n<\/li>\n

  2. \n

    \uacf5\ud1b5\uc124\uc815 \uc801\uc6a9<\/p>\n<\/li>\n

  3. \n

    \uc0dd\uc131\ud55c \uc778\uc2a4\ud134\uc2a4\ub97c \uc774\ubbf8\uc9c0\ub85c \uc800\uc7a5<\/p>\n<\/li>\n

  4. \n

    \uc774\ubbf8\uc9c0\uc5d0\uc11c \uc778\uc2a4\ud134\uc2a4 \ucd94\uac00 \uc0dd\uc131<\/p>\n

    \ucd5c\ucd08 \uc778\uc2a4\ud134\uc2a4\ub97c master \ub85c \ud558\uace0,
    \n3\uac1c\uc758 \uc778\uc2a4\ud134\uc2a4\ub97c \ucd94\uac00\ub85c \uc0dd\uc131\ud558\uc5ec data node \ub85c \uc124\uc815<\/p>\n<\/li>\n<\/ol>\n

    EC2 \uc778\uc2a4\ud134\uc2a4 \ud558\ub098 \uc0dd\uc131<\/h2>\n

    OS \ub294 Ubuntu 20.04 \ub85c \ud569\ub2c8\ub2e4.
    \n\uc778\uc2a4\ud134\uc2a4\ub294 c5a.large \ub85c \uc120\ud0dd\ud569\ub2c8\ub2e4.(\uba54\ubaa8\ub9ac 4\uae30\uac00)<\/p>\n

    \n

    t\uacc4\uc5f4 \uc778\uc2a4\ud134\uc2a4\ubcf4\ub2e4 2\ubc30\uc815\ubcf4 \ube44\uc2f8\uc9c0\ub9cc,
    \n\uc9c0\uc18d\uc801\uc73c\ub85c cpu \ub97c \uc0ac\uc6a9\ud558\ub294 \ub85c\uadf8 \uc2dc\uc2a4\ud15c\uc5d0\uc11c t\uacc4\uc5f4 \uc778\uc2a4\ud134\uc2a4\ub294 \uc704\ud5d8\ud569\ub2c8\ub2e4.<\/p>\n<\/blockquote>\n

    \ubcf4\uc548\uadf8\ub8f9\uc740 \uc774\ub984\uc744 hadoop \uc744 \uc124\uc815\ud558\uace0,
    \n22\ubc88 \ud3ec\ud2b8\ub9cc 0.0.0.0 \uc5d0\uc11c \uc811\uadfc\uc774 \uac00\ub2a5\ud558\ub3c4\ub85d \uc124\uc815\ud569\ub2c8\ub2e4.<\/p>\n

    \uc0dd\uc131\ud55c \uc778\uc2a4\ud134\uc2a4\ub85c \ub85c\uadf8\uc778\ud569\ub2c8\ub2e4.<\/p>\n

    \uacf5\ud1b5\uc124\uc815 \uc801\uc6a9<\/h2>\n

    JDK 1.8 \uc124\uce58<\/h3>\n
    sudo apt-get update\nsudo apt-get install openjdk-8-jdk\njava -version<\/code><\/pre>\n
    hadoop \uacc4\uc815\uc0dd\uc131<\/h3>\n
    sudo groupadd -g 10000 hadoop\nsudo useradd -m -s \/bin\/bash -g hadoop -u 10000 hduser<\/code><\/pre>\n
    hadoop \uacc4\uc815\uc5d0 sudo \uad8c\ud55c \ubd80\uc5ec<\/p>\n
    # \uc5d0\ub514\ud130 vim \uc120\ud0dd\nsudo update-alternatives --config editor\n\nsudo visudo\n......\nhduser ALL=(ALL) NOPASSWD:ALL\n......<\/code><\/pre>\n
    hadoop \uc124\uce58<\/h3>\n
    wget https:\/\/dlcdn.apache.org\/hadoop\/common\/hadoop-3.2.3\/hadoop-3.2.3.tar.gz\nsudo tar xvfz hadoop-3.2.3.tar.gz -C \/usr\/local\/\nsudo chown -R hduser:hadoop \/usr\/local\/hadoop-3.2.3\/<\/code><\/pre>\n
    Spark \uc124\uce58<\/h3>\n
    wget https:\/\/dlcdn.apache.org\/spark\/spark-3.2.1\/spark-3.2.1-bin-hadoop3.2.tgz\nsudo tar xvfz spark-3.2.1-bin-hadoop3.2.tgz -C \/usr\/local\/\nsudo chown -R hduser:hadoop \/usr\/local\/spark-3.2.1-bin-hadoop3.2\/<\/code><\/pre>\n
    hadoop \uacc4\uc815 \ud658\uacbd\uc124\uc815<\/h3>\n
    sudo su - hduser<\/code><\/pre>\n
    vi ~\/.bashrc\n......\nexport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\nexport HADOOP_HOME=\/usr\/local\/hadoop-3.2.3\/\nexport SPARK_HOME=\/usr\/local\/spark-3.2.1-bin-hadoop3.2\/\nexport PATH=$PATH:$JAVA_HOME\/bin:$HADOOP_HOME\/bin\/:$HADOOP_HOME\/sbin:$SPARK_HOME\/bin:$SPARK_HOME\/sbin\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME\/lib\/native\n......\n\nexit\nsudo su - hduser<\/code><\/pre>\n
    \uc608\uc81c \uc2e4\ud589\ud574 \ubcf4\uae30<\/p>\n
    hadoop jar $HADOOP_HOME\/share\/hadoop\/mapreduce\/hadoop-mapreduce-examples-3.2.3.jar wordcount $HADOOP_HOME\/LICENSE.txt ~\/output\/\ncat ~\/output\/part-r-*<\/code><\/pre>\n
    hadoop \uc124\uc815<\/h3>\n
    sudo su - hduser\n\nvi $HADOOP_HOME\/etc\/hadoop\/core-site.xml\n......\n## <configuration> <\/configuration> \uc0ac\uc774\uc5d0 \uc785\ub825\n<property>\n   <name>fs.default.name<\/name>\n   <value>hdfs:\/\/master:9000<\/value>\n<\/property>\n......<\/code><\/pre>\n
    vi $HADOOP_HOME\/etc\/hadoop\/hdfs-site.xml\n......\n## <configuration> <\/configuration> \uc0ac\uc774\uc5d0 \uc785\ub825\n<property>\n   <name>dfs.replication<\/name>\n   <value>2<\/value>\n<\/property>\n<property>\n   <name>dfs.namenode.secondary.http-address<\/name>\n   <value>data01:9868<\/value>\n<\/property>\n<property>\n   <name>dfs.namenode.name.dir<\/name>\n   <value>file:\/home\/hduser\/hadoop\/hadoopdata\/hdfs\/namenode<\/value>\n<\/property>\n<property>\n   <name>dfs.datanode.data.dir<\/name>\n   <value>file:\/home\/hduser\/hadoop\/hadoopdata\/hdfs\/datanode<\/value>\n<\/property>\n......<\/code><\/pre>\n
    vi $HADOOP_HOME\/etc\/hadoop\/yarn-site.xml\n......\n## <configuration> <\/configuration> \uc0ac\uc774\uc5d0 \uc785\ub825\n<property>\n   <name>yarn.nodemanager.aux-services<\/name>\n   <value>mapreduce_shuffle<\/value>\n<\/property>\n<property>\n   <name>yarn.nodemanager.env-whitelist<\/name>\n   <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.hostname<\/name>\n  <value>master<\/value>\n<\/property>\n......<\/code><\/pre>\n
    vi $HADOOP_HOME\/etc\/hadoop\/mapred-site.xml\n......\n## <configuration> <\/configuration> \uc0ac\uc774\uc5d0 \uc785\ub825\n<property>\n   <name>mapreduce.framework.name<\/name>\n   <value>yarn<\/value>\n<\/property>\n<property>\n   <name>mapreduce.application.classpath<\/name>\n   <value>$HADOOP_MAPRED_HOME\/share\/hadoop\/mapreduce\/*:$HADOOP_MAPRED_HOME\/share\/hadoop\/mapreduce\/lib\/*<\/value>\n<\/property>\n......<\/code><\/pre>\n
    vi $HADOOP_HOME\/etc\/hadoop\/hadoop-env.sh\n......\nexport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\n......<\/code><\/pre>\n
    vi $HADOOP_HOME\/etc\/hadoop\/workers\n-------------------\ndata01\ndata02\ndata03\n-------------------<\/code><\/pre>\n
    vi $HADOOP_HOME\/etc\/hadoop\/masters\n-------------------\nmaster\n-------------------<\/code><\/pre>\n
    spark \uc124\uc815<\/h3>\n
    vi $SPARK_HOME\/conf\/slaves\n-------------------\ndata01\ndata02\ndata03\n-------------------<\/code><\/pre>\n
    vi $SPARK_HOME\/conf\/spark-env.sh\n-------------------\nexport JAVA_HOME=\/usr\/lib\/jvm\/java-8-openjdk-amd64\nexport SPARK_MASTER_HOST=master\nexport HADOOP_HOME=\/usr\/local\/hadoop-3.2.3\/\nexport YARN_CONF_DIR=$HADOOP_HOME\/etc\/hadoop\nexport HADOOP_CONF_DIR=$HADOOP_HOME\/etc\/hadoop\n-------------------<\/code><\/pre>\n
    \ube44\ubc00\ubc88\ud638 \uc5c6\ub294 ssh \ub85c\uadf8\uc778 \ud5c8\uc6a9<\/h3>\n
    sudo su - hduser\n\nssh-keygen -t rsa -P '' -f ~\/.ssh\/id_rsa\ncat ~\/.ssh\/id_rsa.pub >> ~\/.ssh\/authorized_keys\nchmod 0600 ~\/.ssh\/authorized_keys<\/code><\/pre>\n
    ssh localhost<\/code><\/pre>\n
    \uc0dd\uc131\ud55c \uc778\uc2a4\ud134\uc2a4\ub97c \uc774\ubbf8\uc9c0\ub85c \uc800\uc7a5<\/h2>\n
    hadoop-spark \ub77c\ub294 \uc774\ub984\uc73c\ub85c \uc774\ubbf8\uc9c0\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4.<\/p>\n
    \"\"<\/p>\n
    \uc774\ubbf8\uc9c0 \uc0dd\uc131\uc5d0\ub294 \uba87\ubd84\uc758 \uc2dc\uac04\uc774 \uac78\ub9bd\ub2c8\ub2e4.<\/p>\n
    \uc778\uc2a4\ud134\uc2a4 \ucd94\uac00\uc0dd\uc131<\/h2>\n
    \uc704\uc5d0\uc11c \uc800\uc7a5\ud55c \uc774\ubbf8\uc9c0\ub85c \uc778\uc2a4\ud134\uc2a4\ub97c 3\uac1c \uc0dd\uc131\ud569\ub2c8\ub2e4.<\/p>\n
    \uc778\uc2a4\ud134\uc2a4\ub294 c5a.large \ub85c \uc120\ud0dd\ud569\ub2c8\ub2e4.(\uba54\ubaa8\ub9ac 4\uae30\uac00)
    \n\ubcf4\uc548\uadf8\ub8f9\uc744 \uc704\uc5d0\uc11c \uc0dd\uc131\ud55c hadoop \uc73c\ub85c \uc120\ud0dd\ud574 \uc90d\ub2c8\ub2e4.<\/p>\n
    \ud638\uc2a4\ud2b8\ud30c\uc77c\uc744 \uc218\uc815\ud569\ub2c8\ub2e4.
    \n\uc778\uc2a4\ud134\uc2a4\uc758 \ud504\ub77c\uc774\ube57 IPv4 \uc8fc\uc18c\ub97c \uc785\ub825\ud574 \uc90d\ub2c8\ub2e4.<\/p>\n
    sudo su - hduser\n\nsudo vi \/etc\/hosts\n-------------------\n172.31.7.69     master\n172.31.7.54     data01\n172.31.6.110    data02\n172.31.2.208    data03\n-------------------<\/code><\/pre>\n
    \uc815\uc0c1 \uc811\uc18d\uc744 \ud655\uc778\ud569\ub2c8\ub2e4.<\/p>\n
    ssh master\nssh data01\nssh data02\nssh data03<\/code><\/pre>\n
    \ud638\uc2a4\ud2b8\ud30c\uc77c\uc744 \ubcf5\uc0ac\ud574 \uc90d\ub2c8\ub2e4.<\/p>\n
    cat \/etc\/hosts | ssh data01 "sudo sh -c 'cat >\/etc\/hosts'"\ncat \/etc\/hosts | ssh data02 "sudo sh -c 'cat >\/etc\/hosts'"\ncat \/etc\/hosts | ssh data03 "sudo sh -c 'cat >\/etc\/hosts'"<\/code><\/pre>\n
    \ubcf4\uc548\uadf8\ub8f9 \uc218\uc815<\/h2>\n
    \ud074\ub7ec\uc2a4\ud130\uac04 \ud2b8\ub798\ud53d\uc744 \ud5c8\uc6a9\ud558\ub3c4\ub85d \ubcf4\uc548\uadf8\ub8f9\uc744 \uc218\uc815\ud574 \uc90d\ub2c8\ub2e4.<\/p>\n
    \"\"<\/p>\n
    hadoop \ubcf4\uc548\uadf8\ub8f9\uba85\uc744 \uc18c\uc2a4\ub85c \uc9c0\uc815\ud574 \uc90d\ub2c8\ub2e4.<\/p>\n
    \ud074\ub7ec\uc2a4\ud130 \uc2dc\uc791\ud558\uae30<\/h2>\n
    \ub124\uc784\ub178\ub4dc \ud3ec\uba67<\/p>\n
    sudo su - hduser\n\nhdfs namenode -format<\/code><\/pre>\n
    start-dfs.sh\n......\nStarting namenodes on [master]\nStarting datanodes\nStarting secondary namenodes [data01]<\/code><\/pre>\n
    master, data01, data02, data03 \uc774 \uc2e4\ud589\ub41c \uac83\uc744 \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
    ssh master\njps\n3125 NameNode\n3420 Jps\nexit\n\nssh data01\njps\n3280 SecondaryNameNode\n3396 Jps\n3119 DataNode\nexit\n\nssh data02\njps\n1113 Jps\n975 DataNode\nexit\n\nssh data03\njps\n1113 Jps\n975 DataNode\nexit<\/code><\/pre>\n
    \ud3f4\ub354 \uc0dd\uc131\uc744 \ud14c\uc2a4\ud2b8 \ud569\ub2c8\ub2e4.<\/p>\n
    hadoop fs -mkdir \/test\nhadoop fs -ls \/<\/code><\/pre>\n
    hdfs dfsadmin -report<\/code><\/pre>\n
    Yarn \uc2e4\ud589<\/h2>\n
    start-yarn.sh<\/code><\/pre>\n
    ssh master\njps\n4050 Jps\n3125 NameNode\n3766 ResourceManager\nexit\n\nssh data01\njps\n3280 SecondaryNameNode\n3825 NodeManager\n4031 Jps\n3119 DataNode\nexit\n\nssh data02\njps\n3777 NodeManager\n2914 DataNode\n3980 Jps\nexit\n\nssh data03\njps\n3588 Jps\n3384 NodeManager\n2909 DataNode\nexit<\/code><\/pre>\n
    yarn node -list\nTotal Nodes:3\n         Node-Id             Node-State Node-Http-Address       Number-of-Running-Containers\n    data01:41077                RUNNING       data01:8042                                  0\n    data02:42445                RUNNING       data02:8042                                  0\n    data03:39255                RUNNING       data03:8042                                  0<\/code><\/pre>\n
    Yarn \ud14c\uc2a4\ud2b8<\/h2>\n
    hadoop fs -put $HADOOP_HOME\/LICENSE.txt \/test\/\n\nyarn jar $HADOOP_HOME\/share\/hadoop\/mapreduce\/hadoop-mapreduce-examples-3.2.3.jar wordcount hdfs:\/\/\/test\/LICENSE.txt \/test\/output\n\nhadoop fs -text \/test\/output\/*<\/code><\/pre>\n
    Spark Shell \uc2e4\ud589\ud558\uae30<\/h2>\n
    spark-shell --master yarn\n......\nscala> :quit<\/code><\/pre>\n
    \ud074\ub7ec\uc2a4\ud130 \uc885\ub8cc\ud558\uae30<\/h2>\n
    stop-yarn.sh\nstop-dfs.sh<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
    Hadoop Cluster \uad6c\uc131\ud558\uae30(with Spark) \ucc38\uc870 : AWS EC2 \ub97c \uc774\uc6a9\ud574 \uad6c\uc131 \ucc38\uc870 : \uac00\uc0c1\uba38\uc2e0\uc744 \uc774\uc6a9\ud574 \uad6c\uc131 \uae30\ubcf8 \ubc29\ud5a5 EC2 \uc778\uc2a4\ud134\uc2a4 \ud558\ub098 \uc0dd\uc131 \uacf5\ud1b5\uc124\uc815 \uc801\uc6a9 \uc0dd\uc131\ud55c \uc778\uc2a4\ud134\uc2a4\ub97c \uc774\ubbf8\uc9c0\ub85c \uc800\uc7a5 \uc774\ubbf8\uc9c0\uc5d0\uc11c \uc778\uc2a4\ud134\uc2a4 \ucd94\uac00 \uc0dd\uc131 \ucd5c\ucd08 \uc778\uc2a4\ud134\uc2a4\ub97c master \ub85c \ud558\uace0, 3\uac1c\uc758 \uc778\uc2a4\ud134\uc2a4\ub97c \ucd94\uac00\ub85c \uc0dd\uc131\ud558\uc5ec data node \ub85c \uc124\uc815 EC2 \uc778\uc2a4\ud134\uc2a4 \ud558\ub098 \uc0dd\uc131 OS \ub294 Ubuntu 20.04 \ub85c \ud569\ub2c8\ub2e4.\u2026 Read More »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[12],"tags":[],"class_list":["post-5184","post","type-post","status-publish","format-standard","hentry","category-devops"],"_links":{"self":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/5184","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=5184"}],"version-history":[{"count":29,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/5184\/revisions"}],"predecessor-version":[{"id":5300,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/5184\/revisions\/5300"}],"wp:attachment":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=5184"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=5184"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=5184"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}