{"id":9959,"date":"2025-02-14T18:41:33","date_gmt":"2025-02-14T09:41:33","guid":{"rendered":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=9959"},"modified":"2025-02-28T19:05:01","modified_gmt":"2025-02-28T10:05:01","slug":"llm%eb%8c%80%ea%b7%9c%eb%aa%a8-%ec%96%b8%ec%96%b4-%eb%aa%a8%eb%8d%b8-large-language-model-%ea%b0%95%ed%99%94%ed%95%99%ec%8a%b5-%ea%b3%bc%ec%a0%95","status":"publish","type":"post","link":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=9959","title":{"rendered":"LLM(\ub300\uaddc\ubaa8 \uc5b8\uc5b4 \ubaa8\ub378, Large Language Model) \uac15\ud654\ud559\uc2b5 \uacfc\uc815"},"content":{"rendered":"<h1>LLM(\ub300\uaddc\ubaa8 \uc5b8\uc5b4 \ubaa8\ub378, Large Language Model) \uac15\ud654\ud559\uc2b5 \uacfc\uc815<\/h1>\n<h2>\uae30\ubcf8 \ubaa8\ub378 \ud559\uc2b5 (Pretraining)<\/h2>\n<p>\uc778\ud130\ub137, \ucc45, \ub17c\ubb38 \ub4f1\uc758 \ub300\ub7c9 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud574 \uc9c0\ub3c4\ud559\uc2b5(Supervised Learning)\uacfc \uc790\uae30\uc9c0\ub3c4\ud559\uc2b5(Self-Supervised Learning) \uc744 \uc218\ud589\ud569\ub2c8\ub2e4.<\/p>\n<p>\uc774 \uacfc\uc815\uc5d0\uc11c \ubaa8\ub378\uc740 \ub2e8\uc5b4, \ubb38\uc7a5, \ubb38\ub9e5\uc744 \ud559\uc2b5\ud558\uace0 \uae30\ubcf8\uc801\uc778 \uc5b8\uc5b4 \ub2a5\ub825\uc744 \uac16\ucd94\uac8c \ub429\ub2c8\ub2e4.<\/p>\n<p>CLM \ud559\uc2b5 \ucc38\uc870<\/p>\n<h2>\uc9c0\ub3c4 \ubbf8\uc138\uc870\uc815 (Supervised Fine-tuning, SFT)<\/h2>\n<p>\uc0ac\ub78c\uc774 \uc9c1\uc811 \ub77c\ubca8\ub9c1\ud55c \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud574 \ubaa8\ub378\uc744 \uc9c0\ub3c4\ud559\uc2b5\ud569\ub2c8\ub2e4.<\/p>\n<p>\uc608\ub97c \ub4e4\uc5b4, \uc9c8\ubb38\uc5d0 \ub300\ud55c \uc801\uc808\ud55c \ub2f5\ubcc0\uc744 \uc0ac\ub78c\uc774 \uc791\uc131\ud558\uace0 \uc774\ub97c \ubaa8\ub378\uc774 \ud559\uc2b5\ud558\ub294 \ubc29\uc2dd\uc785\ub2c8\ub2e4.<\/p>\n<p>\uc9c8\ubb38\uacfc \ub2f5\ubcc0\uc744 \ubaa8\ub450 \uc0ac\ub78c\uc774 \uc791\uc131<\/p>\n<h2>DPO (Direct Preference Optimization)<\/h2>\n<p>\uae30\uc874\uc758 RLHF(\ubcf4\uc0c1 \ud559\uc2b5\uc744 \ud1b5\ud55c \uac15\ud654 \ud559\uc2b5, Reinforcement Learning with Human Feedback) \ub4f1\uc758 \uc811\uadfc\ubc95\uc5d0 \ube44\ud574 \ub354 \uac04\ub2e8\ud558\uba74\uc11c\ub3c4 \ud6a8\uacfc\uc801\uc778 \uacb0\uacfc\ub97c \uc5bb\uc2b5\ub2c8\ub2e4.<\/p>\n<ul>\n<li>\n<p>\uc0ac\uc6a9\uc790 \ud53c\ub4dc\ubc31 \uc218\uc9d1<\/p>\n<p>\ubaa8\ub378\uc774 \uc0dd\uc131\ud55c \ub2e4\uc218\uc758 \ub2f5\ubcc0 \ub610\ub294 \ucd9c\ub825\ubb3c\uc744 \uc0ac\uc6a9\uc790(\uc0ac\ub78c)\uc5d0\uac8c \uc81c\uc2dc\ud558\uace0,<br \/>\n\uac01 \uc30d(pair)\uc5d0 \ub300\ud574 \uc5b4\ub5a4 \ub2f5\ubcc0(\ucd9c\ub825\ubb3c)\uc774 \ub354 \ubc14\ub78c\uc9c1\ud55c\uc9c0\ub97c \ub73b\ud558\ub294 \uc120\ud638 \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud569\ub2c8\ub2e4.<\/p>\n<\/li>\n<li>\n<p>\uc9c1\uc811 \ucd5c\uc801\ud654<\/p>\n<p>\uc120\ud638 \ub370\uc774\ud130\uac00 \ub098\uc62c \ud655\ub960\uc744 \ub192\uc774\uae30 \uc704\ud55c \uc190\uc2e4\ud568\uc218 \uc0dd\uc131<\/p>\n<\/li>\n<li>\n<p>\ud30c\ub77c\ubbf8\ud130 \uc5c5\ub370\uc774\ud2b8<\/p>\n<p>\uc190\uc2e4\ud568\uc218\ub97c \uc774\uc6a9\ud574 \ud30c\ub77c\ubbf8\ud130 \uc5c5\ub370\uc774\ud2b8<\/p>\n<\/li>\n<\/ul>\n<h2>\ubcf4\uc0c1 \ubaa8\ub378 \ud559\uc2b5 (Reward Model Training)<\/h2>\n<p>\ub2e4\uc218\uc758 \ub2f5\ubcc0 \ud6c4\ubcf4\ub97c \uc0dd\uc131\ud55c \ud6c4, \uc0ac\ub78c\uc774 \ub2f5\ubcc0\uc758 \ud488\uc9c8\uc744 \ube44\uad50 \ud3c9\uac00\ud569\ub2c8\ub2e4.<\/p>\n<p>\uc774 \ud3c9\uac00 \ub370\uc774\ud130\ub97c \uae30\ubc18\uc73c\ub85c \ubcf4\uc0c1 \ubaa8\ub378(Reward Model, RM)\uc744 \ud559\uc2b5\ud569\ub2c8\ub2e4.<\/p>\n<p>\ubcf4\uc0c1 \ubaa8\ub378\uc740 &quot;\uc5b4\ub5a4 \ub2f5\ubcc0\uc774 \ub354 \uc88b\uc740\uc9c0&quot;\ub97c \uc608\uce21\ud558\ub294 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4.<\/p>\n<ul>\n<li>\n<p>\uc0ac\ub78c \ub610\ub294 \uc778\uacf5\uc9c0\ub2a5\uc774 \uc9c8\ubb38\uc744 \uc0dd\uc131, \uc0ac\ub78c\uc774 \ud488\uc9c8 \uce21\uc815 (0.0 ~ 1.0 \uc73c\ub85c \uc810\uc218 \ubd80\uc5ec)<\/p>\n<\/li>\n<li>\n<p>\uc9c8\ubb38+\ub2f5\ubcc0+\ud488\uc9c8\uc744 \uc774\uc6a9\ud574 \ubcf4\uc0c1\ubaa8\ub378 \ud559\uc2b5<\/p>\n<p>&quot;\uc9c8\ubb38+\ub2f5\ubcc0&quot; \uc774 \ubcf4\uc0c1 \ubaa8\ub378\uc758 \uc785\ub825\uc774 \ub418\uace0, &quot;\ud488\uc9c8&quot; \uc774 \ucd9c\ub825\uc774 \ub418\uc5b4, \ubaa8\ub378\uc744 \ud559\uc2b5<\/p>\n<\/li>\n<\/ul>\n<h2>\uac15\ud654\ud559\uc2b5 \ub2e8\uacc4 (Reinforcement Learning, RLHF)<\/h2>\n<p>\ubcf4\uc0c1 \ubaa8\ub378\uc744 \ud65c\uc6a9\ud574 \ub77c\ubca8\ub9c1\uc744 \uc790\ub3d9\ud654\ud558\uace0, \ubaa8\ub378\uc744 \uac15\ud654\ud559\uc2b5\ud569\ub2c8\ub2e4.<\/p>\n<p>\ub300\ud45c\uc801\uc778 \ubc29\ubc95\uc774 Proximal Policy Optimization (PPO) \uc785\ub2c8\ub2e4.<\/p>\n<p>\ubaa8\ub378\uc774 \uc5ec\ub7ec \uac1c\uc758 \ub2f5\ubcc0\uc744 \uc0dd\uc131\ud558\uba74 \ubcf4\uc0c1 \ubaa8\ub378\uc774 \uc810\uc218\ub97c \ub9e4\uae30\uace0, \uc774 \uc810\uc218\ub97c \uae30\ubc18\uc73c\ub85c \ubaa8\ub378\uc774 \ub354 \ub098\uc740 \ub2f5\ubcc0\uc744 \uc0dd\uc131\ud558\ub3c4\ub85d \uc5c5\ub370\uc774\ud2b8\ud569\ub2c8\ub2e4.<\/p>\n<p>PPO \ub294 \uc544\ub798\uc5d0\uc11c \ubcc4\ub3c4 \uc124\uba85<\/p>\n<h2>\ucd94\uac00 \ubbf8\uc138\uc870\uc815 \ubc0f \ud3c9\uac00 (Fine-tuning &amp; Evaluation)<\/h2>\n<p>\uc0ac\ub78c\uc774 \uc9c1\uc811 \ub77c\ubca8\ub9c1\ud55c \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud574 \ub2e4\uc2dc \ubaa8\ub378\uc744 \uc9c0\ub3c4\ud559\uc2b5\ud560 \uc218\ub3c4 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n<p>\uc774\ud6c4, \uc0ac\ub78c\uc774 \uc9c8\ubb38\uc744 \uc785\ub825\ud574 \ub2f5\ubcc0\uc744 \ud3c9\uac00\ud569\ub2c8\ub2e4.<\/p>\n<h2>PPO (Proximal Policy Optimization)<\/h2>\n<h3>Policy Model (\uc815\ucc45 \ubaa8\ub378):<\/h3>\n<ul>\n<li>\uc5d0\uc774\uc804\ud2b8\uc758 \ud589\ub3d9\uc744 \uacb0\uc815\ud558\ub294 \ud575\uc2ec \ubaa8\ub378\uc785\ub2c8\ub2e4<\/li>\n<li>\uc785\ub825\uc73c\ub85c \uc0c1\ud0dc(q)\ub97c \ubc1b\uc544 \ud589\ub3d9(o)\uc744 \ucd9c\ub825\ud569\ub2c8\ub2e4<\/li>\n<li>\ud658\uacbd\uacfc \uc9c1\uc811 \uc0c1\ud638\uc791\uc6a9\ud558\uc5ec \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\ub294 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4<\/li>\n<li>\ud559\uc2b5 \uc5ec\ubd80: LLM \uc5d0\uc11c \uc815\ucc45 \ubaa8\ub378 \uc790\uccb4\uac00 \uc778\uacf5\uc9c0\ub2a5 \ubaa8\ub378\uc774 \ub418\uba70, \ud559\uc2b5\uc758 \ubaa9\ud45c\uc785\ub2c8\ub2e4.<\/li>\n<\/ul>\n<h3>Reference Model (\ucc38\uc870 \ubaa8\ub378):<\/h3>\n<ul>\n<li>\uc774\uc804 \uc815\ucc45\uc758 \uc0c1\ud0dc\ub97c \uc800\uc7a5\ud558\ub294 \ubaa8\ub378\uc785\ub2c8\ub2e4<\/li>\n<li>\ud604\uc7ac \uc815\ucc45\uc774 \uc774\uc804 \uc815\ucc45\uc5d0\uc11c \ub108\ubb34 \ub9ce\uc774 \ubc97\uc5b4\ub098\uc9c0 \uc54a\ub3c4\ub85d \uc81c\ud55c\ud558\ub294 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4<\/li>\n<li>KL divergence\ub97c \ud1b5\ud574 \uc815\ucc45 \ubcc0\ud654\ub97c \uc81c\uc5b4\ud569\ub2c8\ub2e4<\/li>\n<li>\ud559\uc2b5 \uc5ec\ubd80: \uc815\ucc45 \ubaa8\ub378\uacfc \ub3d9\uc77c\ud55c \uc0c1\ud0dc\ub85c \uc2dc\uc791\ud558\uba70, \uace0\uc815\ub41c \ubaa8\ub378\uc774 \ub420\uc218\ub3c4, \uc77c\uc815\uc8fc\uae30\ub9c8\ub2e4 \uc5c5\ub370\uc774\ud2b8(\uc815\ucc45\ubaa8\ub378 \ubcf5\uc0ac)\ub418\uae30\ub3c4 \ud569\ub2c8\ub2e4.<\/li>\n<\/ul>\n<h3>Reward Model (\ubcf4\uc0c1 \ubaa8\ub378):<\/h3>\n<ul>\n<li>\uc5d0\uc774\uc804\ud2b8\uc758 \ud589\ub3d9\uc5d0 \ub300\ud55c \ubcf4\uc0c1\uc744 \uacc4\uc0b0\ud558\ub294 \ubaa8\ub378\uc785\ub2c8\ub2e4<\/li>\n<li>\ud589\ub3d9\uc758 \uc801\uc808\uc131\uc744 \ud3c9\uac00\ud558\uace0 \uc218\uce58\ud654\ub41c \ubcf4\uc0c1(r)\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4<\/li>\n<li>\ud559\uc2b5\uc758 \ubc29\ud5a5\uc744 \uacb0\uc815\ud558\ub294 \uc911\uc694\ud55c \uc2e0\ud638\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4<\/li>\n<li>\ud559\uc2b5 \uc5ec\ubd80: \uc77c\ubc18\uc801\uc73c\ub85c \uace0\uc815\ub41c \ubaa8\ub378\ub85c \uc0ac\uc6a9\ub429\ub2c8\ub2e4.<\/li>\n<\/ul>\n<h3>Value Model (\uac00\uce58 \ubaa8\ub378):<\/h3>\n<ul>\n<li>\ud604\uc7ac \uc0c1\ud0dc\uc758 \uae30\ub300 \ubcf4\uc0c1\uc744 \ucd94\uc815\ud558\ub294 \ubaa8\ub378\uc785\ub2c8\ub2e4<\/li>\n<li>\ubbf8\ub798\uc5d0 \uc5bb\uc744 \uc218 \uc788\ub294 \ubcf4\uc0c1\uc758 \ud604\uc7ac \uac00\uce58(v)\ub97c \uacc4\uc0b0\ud569\ub2c8\ub2e4<\/li>\n<li>\uc815\ucc45\uc758 \uc131\ub2a5\uc744 \ud3c9\uac00\ud558\uace0 \uac1c\uc120 \ubc29\ud5a5\uc744 \uc81c\uc2dc\ud558\ub294 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4<\/li>\n<li>\ud559\uc2b5 \uc5ec\ubd80: \ud559\uc2b5 \uac00\ub2a5\ud55c \ubaa8\ub378\ub85c, \uc2e4\uc81c \uc5d0\uc774\uc804\ud2b8\uc758 \ubcf4\uc0c1\uacfc \uc608\uc0c1 \ubcf4\uc0c1 \uac04\uc758 \ucc28\uc774\ub97c \ucd5c\uc18c\ud654\ud558\ub3c4\ub85d \ucd5c\uc801\ud654\ub429\ub2c8\ub2e4.<\/li>\n<\/ul>\n<h2>GRPO (Group Relative Policy Optimization)<\/h2>\n<p>DeepSeek \uc5d0 \uc801\uc6a9\ub41c \uac15\ud654\ud559\uc2b5 \ubc29\uc2dd\uc785\ub2c8\ub2e4.<\/p>\n<p>Value \ubaa8\ub378\uc744 \uc81c\uac70\ud558\uace0, \ubcf4\uc0c1\ubaa8\ub378\uc744 \ub9e4\uc6b0 \ub2e8\uc21c\ud654\ud588\uc2b5\ub2c8\ub2e4.<\/p>\n<p><a href=\"https:\/\/www.skyer9.pe.kr\/wordpress\/wp-content\/uploads\/2025\/02\/2025-02-14-01.png\"><img decoding=\"async\" src=\"https:\/\/www.skyer9.pe.kr\/wordpress\/wp-content\/uploads\/2025\/02\/2025-02-14-01.png\" alt=\"\" \/><\/a><\/p>\n<pre><code class=\"language-python\"># https:\/\/huggingface.co\/docs\/trl\/main\/en\/grpo_trainer\n\n# pip install trl\n\nfrom datasets import load_dataset\nfrom trl import GRPOConfig, GRPOTrainer\n\ndataset = load_dataset(&quot;trl-lib\/tldr&quot;, split=&quot;train&quot;)\n\n# Define the reward function, which rewards completions that are close to 20 characters\ndef reward_len(completions, **kwargs):\n    return [-abs(20 - len(completion)) for completion in completions]\n\ntraining_args = GRPOConfig(output_dir=&quot;Qwen2-0.5B-GRPO&quot;, logging_steps=10)\ntrainer = GRPOTrainer(\n    model=&quot;Qwen\/Qwen2-0.5B-Instruct&quot;,\n    reward_funcs=reward_len,\n    args=training_args,\n    train_dataset=dataset,\n)\ntrainer.train()<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>LLM(\ub300\uaddc\ubaa8 \uc5b8\uc5b4 \ubaa8\ub378, Large Language Model) \uac15\ud654\ud559\uc2b5 \uacfc\uc815 \uae30\ubcf8 \ubaa8\ub378 \ud559\uc2b5 (Pretraining) \uc778\ud130\ub137, \ucc45, \ub17c\ubb38 \ub4f1\uc758 \ub300\ub7c9 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud574 \uc9c0\ub3c4\ud559\uc2b5(Supervised Learning)\uacfc \uc790\uae30\uc9c0\ub3c4\ud559\uc2b5(Self-Supervised Learning) \uc744 \uc218\ud589\ud569\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc5d0\uc11c \ubaa8\ub378\uc740 \ub2e8\uc5b4, \ubb38\uc7a5, \ubb38\ub9e5\uc744 \ud559\uc2b5\ud558\uace0 \uae30\ubcf8\uc801\uc778 \uc5b8\uc5b4 \ub2a5\ub825\uc744 \uac16\ucd94\uac8c \ub429\ub2c8\ub2e4. CLM \ud559\uc2b5 \ucc38\uc870 \uc9c0\ub3c4 \ubbf8\uc138\uc870\uc815 (Supervised Fine-tuning, SFT) \uc0ac\ub78c\uc774 \uc9c1\uc811 \ub77c\ubca8\ub9c1\ud55c \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud574 \ubaa8\ub378\uc744 \uc9c0\ub3c4\ud559\uc2b5\ud569\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4,\u2026 <span class=\"read-more\"><a href=\"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=9959\">Read More &raquo;<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[18],"tags":[],"class_list":["post-9959","post","type-post","status-publish","format-standard","hentry","category-machine-learning"],"_links":{"self":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/9959","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=9959"}],"version-history":[{"count":8,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/9959\/revisions"}],"predecessor-version":[{"id":10061,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/9959\/revisions\/10061"}],"wp:attachment":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=9959"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=9959"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=9959"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}