{"id":1035,"date":"2020-06-28T22:27:14","date_gmt":"2020-06-28T13:27:14","guid":{"rendered":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=1035"},"modified":"2020-06-30T19:06:09","modified_gmt":"2020-06-30T10:06:09","slug":"elasticsearch-%eb%ac%b4%ec%a4%91%eb%8b%a8-%eb%a6%ac%ec%9d%b8%eb%8d%b1%ec%8b%b1","status":"publish","type":"post","link":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=1035","title":{"rendered":"Elasticsearch zero downtime reindex"},"content":{"rendered":"

Elasticsearch zero downtime reindex<\/h1>\n

JDBC \ub97c \uc774\uc6a9\ud55c RDBMS \uc5f0\ub3d9\ud658\uacbd<\/a> \uc5d0\uc11c\uc758 \ubb34\uc911\ub2e8 \ub9ac\uc778\ub371\uc2f1\uc785\ub2c8\ub2e4.<\/p>\n

\ubb34\uc911\ub2e8 \ub9ac\uc778\ub371\uc2f1 \uc804\ub7b5<\/h2>\n
    \n
  1. \n

    \ucd5c\ucd08\uc5d0 testindex_1<\/code> \uc778\ub371\uc2a4\uc640 testindex<\/code> \ub77c\ub294 alias \uac00 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n<\/li>\n

  2. \n

    testindex_\ud0c0\uc784\uc2a4\ud0ec\ud504<\/code> \ub77c\ub294 \uc778\ub371\uc2a4\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4.<\/p>\n<\/li>\n

  3. \n

    reindex<\/code> \ud569\ub2c8\ub2e4.<\/p>\n<\/li>\n

  4. \n

    alias \uac00 \uc0c8\ub85c \uc0dd\uae34 testindex_\ud0c0\uc784\uc2a4\ud0ec\ud504<\/code> \ub97c \ud5a5\ud558\ub3c4\ub85d \ubcc0\uacbd\ud569\ub2c8\ub2e4.<\/p>\n<\/li>\n

  5. \n

    \uc6d0\ub798\uc758 \uc778\ub371\uc2a4\ub294 \uc0ad\uc81c\ud569\ub2c8\ub2e4.<\/p>\n<\/li>\n

  6. \n

    reindex \ub3c4\uc911\uc5d0 \ubc1c\uc0dd\ud558\ub294 update \uac00 \ub204\ub77d\ub420 \uc218 \uc788\uc73c\ubbc0\ub85c \ub514\ube44\ub97c \uc5c5\ub370\uc774\ud2b8 \ud574\uc11c \ubcc0\uacbd\ubd84\uc744 \uc0c8\ub85c \uc0dd\uae34 \uc778\ub371\uc2a4\uc5d0 \uc7ac\uc804\uc1a1\ud558\ub3c4\ub85d \ud569\ub2c8\ub2e4.<\/p>\n<\/li>\n<\/ol>\n

    python \ud655\uc778<\/h2>\n
    python3 -V\nPython 3.6.9\n\nsudo apt install python3-pip\n\npip3 install elasticsearch\npip3 install pymysql\n\npip3 install --upgrade requests<\/code><\/pre>\n
    mappings.json \uc0dd\uc131<\/h2>\n
    cat mappings.json<\/code><\/pre>\n
    {\n  "settings" : {\n    "number_of_shards": "5",\n    "number_of_replicas": "1",\n    "index":{\n      "analysis":{\n        "analyzer":{\n          "korean":{\n            "type":"custom",\n            "tokenizer":"seunjeon_default_tokenizer",\n            "filter" : ["lowercase", "synonym", "stopword"]\n          },\n          "whitespace": {\n            "type": "custom",\n            "tokenizer": "whitespace",\n            "char_filter": [\n              "sepcial_char_filter"\n            ],\n            "filter": [\n              "lowercase",\n              "stop",\n              "snowball"\n            ]\n          },\n          "standard": {\n            "type": "custom",\n            "tokenizer": "standard",\n            "char_filter": [\n              "sepcial_char_filter"\n            ],\n            "filter": [\n              "lowercase",\n              "stop",\n              "snowball"\n            ]\n          }\n        },\n        "char_filter": {\n          "sepcial_char_filter": {\n            "type": "mapping",\n            "mappings": [ "+ => _plus_", "- => _minus_", "# => _sharp_" ]\n          }\n        },\n        "filter" : {\n          "synonym" : {\n            "type" : "synonym",\n            "synonyms_path" : "synonyms.txt"\n          },\n          "stopword" : {\n            "type" : "stop",\n            "stopwords_path" : "stopwords.txt"\n          }\n        },\n        "tokenizer": {\n          "seunjeon_default_tokenizer": {\n            "index_eojeol": "true",\n            "user_dict_path": "user_dict.csv",\n            "index_poses": [\n                "UNK", "EP", "I", "J", "M",\n                "N", "SL", "SH", "SN", "VCP",\n                "XP", "XS", "XR"\n            ],\n            "decompound": "true",\n            "type": "seunjeon_tokenizer"\n          }\n        }\n      }\n    }\n  },\n  "mappings" : {\n    "_doc" : {\n      "properties" : {\n        "@timestamp" : {\n          "type" : "date"\n        },\n        "itemid" : {\n          "type" : "integer"\n        },\n        "itemname" : {\n          "type" : "text",\n          "fields": {\n            "en": {\n              "type": "text",\n              "analyzer": "standard"\n            },\n            "ko": {\n              "type": "text",\n              "analyzer": "korean"\n            }\n          }\n        },\n        "category" : {\n          "type" : "text",\n          "analyzer": "korean"\n        },\n        "price" : {\n          "type": "scaled_float",\n          "scaling_factor": 10000\n        },\n        "lastupdate" : {\n          "type" : "date"\n        },\n        "regdate" : {\n          "type" : "date"\n        }\n      }\n    }\n  }\n}<\/code><\/pre>\n
    python \ud30c\uc77c \uc0dd\uc131<\/h2>\n
    cat reindex.py<\/code><\/pre>\n
    from elasticsearch import Elasticsearch\nimport pymysql\nimport json\nfrom datetime import datetime\n\ndef _suffix():\n    return datetime.now().strftime("%Y%m%d%H%M%S%f")\n\nes = Elasticsearch('localhost:9200')\nindex = 'items_1'\nalias = 'items'\n\nwith open('mappings.json', 'r') as f:\n    mappings = json.load(f)\n\n# if es.indices.exists(index=index):\n#     pass\n# else:\n#     es.indices.create(index=index, body=mappings)\n#     es.indices.update_aliases({\n#         "actions": [\n#             {"add": {"index": index, "alias": alias}}\n#         ]\n#     })\n\nold_index, new_index = '', ''\nfor idx in es.indices.get('items*'):\n    if old_index == '':\n        old_index = idx\n    elif new_index == '':\n        new_index = idx\n\n    if old_index != '' and new_index != '':\n        print('reindexing in process...')\n        quit()\n\nnew_index = '%s_%s' % (alias, _suffix())\nes.indices.create(index=new_index, body=mappings)\n\nnow = datetime.now()\n\nresult = es.reindex({\n        "source": { "index": old_index },\n        "dest": { "index": new_index }\n    }, wait_for_completion=True, request_timeout=600)\n\nprint('reindex finished.')\n\nes.indices.update_aliases({\n    "actions": [\n        {"remove": { "index": old_index, "alias": alias }},\n        {"add": { "index": new_index, "alias": alias }}\n    ]\n})\n\nes.indices.delete(index=old_index, ignore=[400, 404])\n\nconn = pymysql.connect(host='localhost', user='testuser', password='test1234',\n                       db='db_test', charset='utf8')\n\nsql = "update items set lastupdate = now() where lastupdate >= %s"\n\ncurs = conn.cursor()\ncurs.execute(sql, (now))\nconn.commit()\n\nconn.close()<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
    Elasticsearch zero downtime reindex JDBC \ub97c \uc774\uc6a9\ud55c RDBMS \uc5f0\ub3d9\ud658\uacbd \uc5d0\uc11c\uc758 \ubb34\uc911\ub2e8 \ub9ac\uc778\ub371\uc2f1\uc785\ub2c8\ub2e4. \ubb34\uc911\ub2e8 \ub9ac\uc778\ub371\uc2f1 \uc804\ub7b5 \ucd5c\ucd08\uc5d0 testindex_1 \uc778\ub371\uc2a4\uc640 testindex \ub77c\ub294 alias \uac00 \uc788\uc2b5\ub2c8\ub2e4. testindex_\ud0c0\uc784\uc2a4\ud0ec\ud504 \ub77c\ub294 \uc778\ub371\uc2a4\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4. reindex \ud569\ub2c8\ub2e4. alias \uac00 \uc0c8\ub85c \uc0dd\uae34 testindex_\ud0c0\uc784\uc2a4\ud0ec\ud504 \ub97c \ud5a5\ud558\ub3c4\ub85d \ubcc0\uacbd\ud569\ub2c8\ub2e4. \uc6d0\ub798\uc758 \uc778\ub371\uc2a4\ub294 \uc0ad\uc81c\ud569\ub2c8\ub2e4. reindex \ub3c4\uc911\uc5d0 \ubc1c\uc0dd\ud558\ub294 update \uac00 \ub204\ub77d\ub420 \uc218 \uc788\uc73c\ubbc0\ub85c \ub514\ube44\ub97c \uc5c5\ub370\uc774\ud2b8 \ud574\uc11c \ubcc0\uacbd\ubd84\uc744 \uc0c8\ub85c \uc0dd\uae34 \uc778\ub371\uc2a4\uc5d0\u2026 Read More »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[6],"tags":[],"class_list":["post-1035","post","type-post","status-publish","format-standard","hentry","category-elasticsearch"],"_links":{"self":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/1035","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1035"}],"version-history":[{"count":13,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/1035\/revisions"}],"predecessor-version":[{"id":1069,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/1035\/revisions\/1069"}],"wp:attachment":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1035"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1035"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1035"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}