【数据库】es-Toy模板网

这篇具有很好参考价值的文章主要介绍了【数据库】es。希望对大家有所帮助。如果存在错误或未考虑完全的地方，请大家不吝赐教，您也可以点击"举报违法"按钮提交疑问。

新增一条数据:

PUT /xkroot_211111/xkrootType/XqzCv3sBadUEyhUoGKsA
{
  "investigationNo":"W350581B013002021100004",
  "userId":"yNRP2Oe0EeicEgAMKdpT_w",
  "orgCode":"705420347",
  "deviceType":"采集设备名称",
  "companyName":""
}

编辑一条数据：

POST /op_210513/opType/XqzCv3sBadUEyhUoGKsA/_update?pretty
{
  "doc": {
    "appSource":1
  }
}

分组查询：

Map<String, Long> reqLenMap = new HashMap<>();
        QueryBuilder filterBuilder = QueryBuilders.boolQuery()
                .must(QueryBuilders.termsQuery(EsConstants.COLLECT_ID, collectIds))
                .must(QueryBuilders.termsQuery(EsConstants.INTFTP, Arrays.asList("XK", "WJ")))
                .must(QueryBuilders.termQuery(EsConstants.LOGTP, BaseConstants.ONE));
        NativeSearchQueryBuilder queryBuilder = new NativeSearchQueryBuilder()
                .withQuery(filterBuilder)
                .withIndices(EsConstants.INDEX_CSLOG)
                .withSize(BaseConstants.ZERO)
                .addAggregation(
                        AggregationBuilders
                                .terms("cltIdTerms")
                                .field(EsConstants.COLLECT_ID)
                                .shardSize(Integer.MAX_VALUE)
                                .size(Integer.MAX_VALUE)
                                .subAggregation(
                                        AggregationBuilders.sum("lenTerms")
                                        .field(EsConstants.REQLEN)
                                )
                                .shardSize(Integer.MAX_VALUE)
                                .size(Integer.MAX_VALUE)
                );
        SimpleAggregations simpleAggregations = esTemplate.queryForSimpleAggregations(queryBuilder.build());
        if (!simpleAggregations.asMap().isEmpty()) {
            SimpleAggregation simpleAggregation = simpleAggregations.asMap().get("cltIdTerms");
            List<Bucket> buckets = simpleAggregation.getBuckets();
            buckets.forEach(e -> {
                String collectId = e.getKey();
                double reqlen = Double.valueOf(e.getSubAggregation().get("lenTerms").getValue());
                reqLenMap.put(collectId, (long) reqlen);
            });
        }

{
  "from" : 0,
  "size" : 0,
  "query" : {
    "bool" : {
      "must" : [
        {
          "terms" : {
            "collectId" : [
              "11111"
            ],
            "boost" : 1.0
          }
        },
        {
          "terms" : {
            "intfTp" : [
              "XK",
              "WJ"
            ],
            "boost" : 1.0
          }
        },
        {
          "term" : {
            "logTp" : {
              "value" : 1,
              "boost" : 1.0
            }
          }
        }
      ],
      "disable_coord" : false,
      "adjust_pure_negative" : true,
      "boost" : 1.0
    }
  },
  "aggregations" : {
    "cltIdTerms" : {
      "terms" : {
        "field" : "collectId",
        "size" : 2147483647,
        "shard_size" : 2147483647,
        "min_doc_count" : 1,
        "shard_min_doc_count" : 0,
        "show_term_doc_count_error" : false,
        "order" : [
          {
            "_count" : "desc"
          },
          {
            "_term" : "asc"
          }
        ]
      },
      "aggregations" : {
        "lenTerms" : {
          "sum" : {
            "field" : "reqLen"
          }
        }
      }
    }
  },
  "ext" : { }
}

更新ElasticSearch:

Map<String,Object> source = new HashMap<>();
source.put(EsConstants.APP_QRCODE_STR, fromUrl);
UpdateRequest request = new UpdateRequest();
request.doc(source);
UpdateQuery updateQuery = new UpdateQueryBuilder().withId(bdDevInstall.getId())
        .withIndexName(bdDevInstall.get_index()).withType(bdDevInstall.get_type()).withUpdateRequest(request).build();
elasticsearchTemplate.update(updateQuery);
elasticsearchTemplate.refresh(EsConstants.INDEX_OP);

2.创建别名索引：

1）创建别名索引

curl -XPUT “http://172.16.73.33:9200/_template/csmulog_template?pretty” -d @./csmulog_template.json -H ‘Content-Type:application/json’

2）编辑模板文件json

3）索引表要加上

curl -i -k -H “Content-type:application/json” -X POST “http://ip:9200/gg_220101/ggType/OphvM34B5mv9fksiv_Mo/_update?pretty” -d ‘{“doc”: {“cDisks”:64}}’

3.修改索引mapping

curl -XPUT “http://172.16.32.138:9200/_template/csmulog_template?pretty” -d @./csmulog_template.json -H ‘Content-Type:application/json’

Operator Filter{fieldName=‘mnSt’, operator=‘$in’, value=[http://99995999.com]}

Operator Filter{fieldName=‘mnSt’, operator=‘$in’, value=[[http://99995999.com]]}

[http://99995999.com]

[eee614ae567583bb31980400a15f9c8c]

[http://99995999.com]

And Filter{filters=[Operator Filter{fieldName=‘mnSt’, operator=‘$in’, value=[[http://99995999.com]]}]}

4.语句

GET /xkroot/xkrootType/_search
{
  "query":{
    "match_all":{
      
    }
  }
}

PUT /xkroot_211111/xkrootType/XqzCv3sBadUEyhUoGKsA
{
  "investigationNo":"W350581B013002021100004",
  "userId":"yNRP2Oe0EeicEgAMKdpT_w",
  "orgCode":"705420347",
  "deviceType":"采集设备名称",
  "companyName":""
}
PUT /xkroot_211111/xkrootType/XqzCv3sBadUEyhUoGKsA
{
      "infoRegisterType": "01",
      "collectTargetId": "350103240000120768164710152600007934645338662470573827720",
      "srcId": "rzdEdPku4rmFPIRni0R1rQ",
      "isMainWeb": "0",
      "fileMd5": "d5c2a1fdbe3b64c31e98b135522f2398",
      "url": "http://99995999.com",
      "domainId": "9a076877beabcaeb6090890a6fe08b09",
      "collectId": "d6rdv9rmWN8nAA8GhWWKMA",
      "collectTime": "2022-03-12T08:12:06.000Z",
      "appType": "APP|URL",
      "topAppType": "APP",
      "pAppType": "APP",
      "table": "BdAppUrl"
}

POST /xkroot_211111/xkrootType/XqzCv3sBadUEyhUoGKsA/_update?pretty
{
  "doc": {
    "id":"XqzCv3sBadUEyhUoGKsA"
  }
}

POST /csmulog/_search
{
  "query": {
    "match_all": {} 
  },
  "sort": [
    {
      "updateTime": {
        "order": "desc"
      }
    }
  ],
  "size": 200
}

GET _search
{
  "query": {
    "bool": {
      "must": [
        {
          "term": {
            "table": "GgCollectObject"
          }
        }
      ],
      "must_not": [
        {
          "exists":{
            "field":"money"
          }
        }
      ]
    }
  },
  "size": 2000
}
GET _search
{
  "query": {
    "bool": {
      "must": [
        {
          "term": {
            "table": "BdAppNetworkBehavior"
          }
        },
        {
          "term": {
            "collectId": "MrF4qrWsHsq1Kqqyr9wscg"
          }
        },{
          "term": {
            "appMd5": {
              "value": "419d7e3c79dbe75c82b5b34d4f90e45a"
            }
          }
        }
      ]
    }
  }
}
GET /cslog/_search
{
  "query": {
    "match_all": {}
  },
  "size": 20,
  "sort": [
    {
      "updateTime": {
        "order": "desc"
      }
    }
  ]
}

GET _template

PUT _template/csmulog_template
{
    "order": 0,
    "index_patterns": [
      "csmulog_*"
    ],
    "settings": {
      "index": {
        "refresh_interval": "30s",
        "unassigned": {
          "node_left": {
            "delayed_timeout": "7d"
          }
        },
        "analysis": {
          "analyzer": {
            "bigrams": {
              "filter": "cjk_bigram",
              "tokenizer": "standard"
            }
          }
        },
        "translog": {
          "flush_threshold_size": "1.5gb",
          "sync_interval": "2m",
          "durability": "async"
        },
        "number_of_replicas": "0",
        "merge": {
          "scheduler": {
            "max_thread_count": "1"
          },
          "policy": {
            "max_merge_at_once": "15"
          }
        }
      }
    },
    "mappings": {
      "csmulogType": {
        "_all": {
          "analyzer": "bigrams"
        },
        "dynamic_templates": [
          {
            "long_count": {
              "mapping": {
                "doc_values": false,
                "type": "long"
              },
              "match": "*_count",
              "match_mapping_type": "long"
            }
          },
          {
            "string_name": {
              "mapping": {
                "doc_values": false,
                "include_in_all": true,
                "type": "text"
              },
              "match": "*_name",
              "match_mapping_type": "string"
            }
          },
          {
            "string_num": {
              "mapping": {
                "doc_values": false,
                "include_in_all": true,
                "type": "text"
              },
              "match": "*_num",
              "match_mapping_type": "string"
            }
          },
          {
            "double_price": {
              "mapping": {
                "doc_values": false,
                "type": "double"
              },
              "match": "*_price",
              "match_mapping_type": "double"
            }
          },
          {
            "date_time": {
              "mapping": {
                "doc_values": false,
                "type": "date"
              },
              "match": "*_time",
              "match_mapping_type": "object"
            }
          },
          {
            "string_bak": {
              "mapping": {
                "doc_values": false,
                "index": "no",
                "type": "keyword"
              },
              "match": "*_bak",
              "match_mapping_type": "string"
            }
          },
          {
            "string_text": {
              "mapping": {
                "include_in_all": true,
                "type": "text"
              },
              "match": "text_*",
              "match_mapping_type": "string"
            }
          }
        ],
        "include_in_all": false,
        "properties": {
          "taskName": {
            "type": "keyword"
          },
          "tableName": {
            "type": "keyword"
          },
          "recordId": {
            "type": "keyword"
          },
          "taskId": {
            "type": "keyword"
          },
          "collectId": {
            "type": "keyword"
          },
          "caseId": {
            "type": "text"
          },
          "oldValue": {
            "type": "text"
          },
          "newValue": {
            "type": "text"
          },
          "operateType": {
            "type": "integer"
          },
          "userIP": {
            "type": "text"
          },
          "userId": {
            "type": "text"
          },
          "updateTime": {
            "type": "date"
          }
        }
      }
    },
    "aliases": {
      "csmulog_query": {},
      "csmulog": {}
    }
  }
  
GET /_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "table": "GjHandsetCall"
          }
        },
        {
          "exists": {
            "field": "startTime"
          }
        }
      ],
      "must_not": [
        {
          "terms": {
            "collectId": [
              "U3o3V6o9CgPqVn236Bp3oA",
              "Kd4C7q3alMdWb-2DPiWFrA",
              "GxKNx0yHg8uw8PRkvJVpvA",
              "It6tixIf6X8l4dpadImKKA",
              "a56FE4JudL_11OrxWsaIRw",
              "5PZm-DCCmHKvIqa1blwEhg",
              "VYowEOyGLrrnclMs-x_frQ",
              "jw8NgFUyAFvvDFB04zPGgA",
              "WuCLBo3bGVKItcRWjKM9xA"
            ]
          }
        }
      ]
    }
  }
}
  
  
GET /gg/_search
{
  "query": {
    "term": {
      "table": {
        "value": "GgScoutInfo"
      }
    }
  },
  "size": 20,
  "sort": [
    {
      "collectTime": {
        "order": "desc"
      }
    }
  ]
}

PUT 172.16.32.138:9200/slog_220111

5.u01空间满了之后，索引自动变为只读

清理空间之后执行

curl -XPUT -H “Content-Type:application/json” http://172.16.32.51:9200/_all/_settings -d ‘{“index.blocks.read_only_allow_delete”:null}’

6.问题分组

1.min_doc_count：在query的json中，可以用于限制分组的数量最少多少文章来源地址https://www.toymoban.com/news/detail-841208.html

GET hs/_search
{
	"query":{
        "bool":{
            "must":[
                {
                    "term":{
                        "table":{
                            "value":"BdHandSetCall"
                        }
                    }
                }
            ]
        }
    },
    "aggs":{
        "cltIdTerms":{
            "terms":{
                "field":"collectId"
            }
        },
        "aggs":{
            "terms":{
                "field":"friendAccount",
                "min_doc_count":5						// 至少通联过5次								
            }
        }
    }
}

7.reindex

背景：索引的模板一开始就构建好了，后面改不了，所以重新构建一个索引，然后数据迁移过去

方式：

关闭storm入库服务

一、删除旧template，如果没有xpack插件控制权限，删除命令中的-u 'root':'密码' 
curl -XDELETE -u 'root':'密码' '172.16.2.245:9200/_template/op_template?pretty'

 

二、创建新template，当前目录下放新template的json文件op.json
curl -XPUT -u 'root':'密码' 'http://172.16.2.245:9200/_template/op_template?pretty' -d @./op.json

 

三、创建新索引
curl -XPUT -u 'root':'密码' 'http://172.16.2.245:9200/op_211019_2?pretty'

 

四、移数据
curl -XPOST "http://spider01:9200/_reindex" -H 'Content-Type: application/json' -d'
{
  "source":{
    "index":"op_211019"
  },
  "dest":{
    "index":"op_211019_2"
  }
}'

 

五、移动完成后，修改sys_es_index、sys_es_index_record表，保证数据写入新的index
update sys_es_index set name = 'op_211019_2' where alias = 'op';
update sys_es_index_record set name = 'op_211019_2' where alias = 'op';
 

 

启动storm入库服务

8.集群健康red

1、	分析red原因
执行：GET _cluster/allocation/explain?pretty
返回：
	{
      "node_id": "LPSP4kBjRJmHUGnn80tdag",
      "node_name": "elasticsearch2",
      "transport_address": "172.16.31.113:9300",
      "node_attributes": {
        "ml.machine_memory": "33566740480",
        "xpack.installed": "true",
        "transform.node": "true",
        "ml.max_open_jobs": "512",
        "ml.max_jvm_size": "2077753344"
      },
      "node_decision": "no",
      "store": {
        "in_sync": false,
        "allocation_id": "uY0ygDqEQEeUX4E5MRLKkw"
      }
    },
原因：节点决策是no，集群没有分配存储节点。in_sync是false，数据同步也有异常

2、进行reroute重新强制分片，找不到的数据
POST _cluster/reroute?pretty
{
	"commands": [
		{
			"allocate_stale_primary": {
				"index": ".ds-.logs-deprecation.elasticsearch-default-2022.12.13-000007",
				"shard": 0,
				"node": "elasticsearch2",
				"accept_data_loss": true
			}
		}
	]
}

到了这里，关于【数据库】es的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！