es报错no_shard_available_action_exception

民工心事  论坛元老 | 2024-10-11 17:05:20 | 来自手机 | 显示全部楼层 | 阅读模式
打印 上一主题 下一主题
楼主

主题 1644|帖子 1644|积分 4932

马上注册，结交更多好友，享用更多功能，让你轻松玩转社区。

您需要 登录 才可以下载或查看，没有账号？立即注册

x
一、题目描述
机房迁移之后，发现ES连接报错，查询某个索引数据
  1. GET  http://IP:PORT/XXX_XXX_XXX/_search
复制代码
报错
  1. {
  2.     "error": {
  3.         "root_cause": [
  4.             {
  5.                 "type": "no_shard_available_action_exception",
  6.                 "reason": null,
  7.                 "index_uuid": "xxxxx-xxxx-ddddd",
  8.                 "shard": "0",
  9.                 "index": "XXX_XXX_XXX"
  10.             }
  11.         ],
  12.         "type": "search_phase_execution_exception",
  13.         "reason": "all shards failed",
  14.         "phase": "query",
  15.         "grouped": true,
  16.         "failed_shards": [
  17.             {
  18.                 "shard": 0,
  19.                 "index": "XXX_XXX_XXX",
  20.                 "node": null,
  21.                 "reason": {
  22.                     "type": "no_shard_available_action_exception",
  23.                     "reason": null,
  24.                     "index_uuid": "xxxxx-xxxx-ddddd",
  25.                     "shard": "0",
  26.                     "index": "XXX_XXX_XXX"
  27.                 }
  28.             }
  29.         ]
  30.     },
  31.     "status": 503
  32. }
复制代码
ES日志
  1. {"type": "server", "timestamp": "2024-07-05T09:46:33,177Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "docker-cluster", "node.name": "esXXXX-XXXXXXXXXXXXXX-XXXX", "message": "path: /XXX_XXX_XXX/_search, params: {index=XXX_XXX_XXX}", "cluster.uuid": "_VxVNRD-QB-DDDDDDDDDDD", "node.id": "KdIfLm7ERZGankl-XSWWWWWWWWWWW" ,
  2. "stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",
  3. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:663) [elasticsearch-7.12.1.jar:7.12.1]",
  4. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) [elasticsearch-7.12.1.jar:7.12.1]",
  5. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:696) [elasticsearch-7.12.1.jar:7.12.1]",
  6. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) [elasticsearch-7.12.1.jar:7.12.1]",
  7. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:292) [elasticsearch-7.12.1.jar:7.12.1]",
  8. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:226) [elasticsearch-7.12.1.jar:7.12.1]",
  9. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:424) [elasticsearch-7.12.1.jar:7.12.1]",
  10. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:184) [elasticsearch-7.12.1.jar:7.12.1]",
  11. "at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:668) [elasticsearch-7.12.1.jar:7.12.1]",
  12. "at org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:489) [elasticsearch-7.12.1.jar:7.12.1]",
  13. "at org.elasticsearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:283) [elasticsearch-7.12.1.jar:7.12.1]",
  14. "at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117) [elasticsearch-7.12.1.jar:7.12.1]",
  15. "at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:103) [elasticsearch-7.12.1.jar:7.12.1]",
  16. "at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:76) [elasticsearch-7.12.1.jar:7.12.1]",
  17. "at org.elasticsearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:324) [elasticsearch-7.12.1.jar:7.12.1]",
  18. "at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:212) [elasticsearch-7.12.1.jar:7.12.1]",
  19. "at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:91) [elasticsearch-7.12.1.jar:7.12.1]",
  20. "at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.12.1.jar:7.12.1]",
  21. "at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:42) [elasticsearch-7.12.1.jar:7.12.1]",
  22. "at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) [elasticsearch-7.12.1.jar:7.12.1]",
  23. "at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.12.1.jar:7.12.1]",
  24. "at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.12.1.jar:7.12.1]",
  25. "at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.12.1.jar:7.12.1]",
  26. "at org.elasticsearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:80) [elasticsearch-7.12.1.jar:7.12.1]",
  27. "at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.12.1.jar:7.12.1]",
  28. "at org.elasticsearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:118) [elasticsearch-7.12.1.jar:7.12.1]",
  29. "at org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:106) [elasticsearch-7.12.1.jar:7.12.1]",
  30. "at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:247) [elasticsearch-7.12.1.jar:7.12.1]",
  31. "at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:329) [elasticsearch-7.12.1.jar:7.12.1]",
  32. "at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:180) [elasticsearch-7.12.1.jar:7.12.1]",
  33. "at org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:325) [elasticsearch-7.12.1.jar:7.12.1]",
  34. "at org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:390) [elasticsearch-7.12.1.jar:7.12.1]",
  35. "at org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:307) [elasticsearch-7.12.1.jar:7.12.1]",
  36. "at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:31) [transport-netty4-client-7.12.1.jar:7.12.1]",
  37. "at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:17) [transport-netty4-client-7.12.1.jar:7.12.1]",
  38. "at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  39. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  40. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  41. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  42. "at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:47) [transport-netty4-client-7.12.1.jar:7.12.1]",
  43. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  44. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  45. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  46. "at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  47. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  48. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  49. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  50. "at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  51. "at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  52. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  53. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  54. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  55. "at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  56. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  57. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  58. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  59. "at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  60. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  61. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  62. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  63. "at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  64. "at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  65. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  66. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  67. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  68. "at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
  69. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  70. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  71. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  72. "at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
  73. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  74. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  75. "at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  76. "at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  77. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  78. "at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  79. "at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  80. "at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  81. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  82. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  83. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  84. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
  85. "at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
  86. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
  87. "at java.lang.Thread.run(Thread.java:831) [?:?]",
  88. "Caused by: org.elasticsearch.action.NoShardAvailableActionException",
  89. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:500) ~[elasticsearch-7.12.1.jar:7.12.1]",
  90. "at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:449) [elasticsearch-7.12.1.jar:7.12.1]",
  91. "... 81 more"] }
复制代码
二、题目排查
第一步:检察分片状态
  1. GET http://IP：PORT/_cat/shards?pretty
复制代码
返回值
  1. wcm                   0 p STARTED     0   208b xxx.xxx.xxx.xxx esuuu-iiiiiiiiiiiiiii-hwwwwww
  2. test                  0 p STARTED    21 37.3kb xxx.xxx.xxx.xxx esuuu-iiiiiiiiiiiiiii-hwwwwww
  3. XXX_XXX_XXX           0 p UNASSIGNED                          
  4. cluster               0 p STARTED     2  5.7kb xxx.xxx.xxx.xxx esuuu-iiiiiiiiiiiiiii-hwwwwww
  5. cluster               0 r UNASSIGNED         
  6.   
复制代码
XXX_XXX_XXX分片状态是UNASSIGNED，不正常
第二步：查询分片状态unassigned的缘故原由
  1. http://IP：PORT/_cluster/allocation/explain?pretty  GET
复制代码
结果
  1. {
  2.     "index": "XXX_XXX_XXX",
  3.     "shard": 0,
  4.     "primary": true,
  5.     "current_state": "unassigned",
  6.     "unassigned_info": {
  7.         "reason": "CLUSTER_RECOVERED",
  8.         "at": "2024-07-05T09:39:01.951Z",
  9.         "last_allocation_status": "no_valid_shard_copy"
  10.     },
  11.     "can_allocate": "no_valid_shard_copy",
  12.     "allocate_explanation": "cannot allocate because all found copies of the shard are either stale or corrupt",
  13.     "node_allocation_decisions": [
  14.         {
  15.             "node_id": "KdIfLm7ERZGankl-7b_jtw",
  16.             "node_name": "esuuu-iiiiiiiiiiiiiii-hwwwwww",
  17.             "transport_address": "xxx.xxx.xxx.xxx:9300",
  18.             "node_attributes": {
  19.                 "ml.machine_memory": "2147483648",
  20.                 "xpack.installed": "true",
  21.                 "transform.node": "true",
  22.                 "ml.max_open_jobs": "20",
  23.                 "ml.max_jvm_size": "1073741824"
  24.             },
  25.             "node_decision": "no",
  26.             "store": {
  27.                 "in_sync": true,
  28.                 "allocation_id": "ahagagagagagaga",
  29.                 "store_exception": {
  30.                     "type": "corrupt_index_exception",
  31.                     "reason": "failed engine (reason: [search execution corruption failure]) (resource=preexisting_corruption)",
  32.                     "caused_by": {
  33.                         "type": "i_o_exception",
  34.                         "reason": "failed engine (reason: [search execution corruption failure])",
  35.                         "caused_by": {
  36.                             "type": "corrupt_index_exception",
  37.                             "reason": "Corrupted: docID=147, docBase=15, chunkDocs=11, numDocs=556 (resource=NIOFSIndexInput(path="/usr/share/elasticsearch/data/nodes/0/indices/xhxhxhxxhxhxhxxhxhxx-pMxOzkdhdw/0/index/_b.fdt"))"
  38.                         }
  39.                     }
  40.                 }
  41.             }
  42.         }
  43.     ]
  44. }
复制代码
三、缘故原由分析
节点长时间掉线后重新加入集群，引入了脏数据。
四、解决办法
我尝试了重新分配分片、将一个空分片分配为主分片、重启ES等方法，结果是没有结果，且返回值显示找不到索引对应的数据。可能数据已经丢失。单独查询索引XXX_XXX_XXX，发现索引还在。
终极，我删除了索引，重新同步了一份数据到ES中。
五、参考
https://www.cnblogs.com/qcloud1001/p/13755469.html
https://blog.csdn.net/vinegar93/article/details/135617013

免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。
回复

使用道具 举报

0 个回复

倒序浏览
返回列表

快速回复

高级模式
B Color Image Link Quote Code Smilies
您需要登录后才可以回帖 登录 or 立即注册

本版积分规则

民工心事

论坛元老
这个人很懒什么都没写!

楼主热帖

标签云

集成商 AI 运维 CIO 存储 服务器
微信订阅号
微信服务号
微信客服
小程序
H5
关于我们 商务合作 网站地图
©Copyright 2022-2024 杭州祥升科技有限公司 版权所有 浙ICP备20004199号
快速回复 返回顶部 返回列表