elasticsearch python 주말에 정리할 것
참고 url
https://www.youtube.com/watch?v=UPkqFvjN-yI
from elasticsearch import Elasticsearch
import json
from yaml import load
class Elaserv:
def __init__(self):
self.client = Elasticsearch(hosts=Elaserv.yamlFileRead())
"""
matchall query 에 대한 함수
"""
def defaultMatchAll(self):
queryJson = open("./ddl/match-all.json", "r")
query = json.load(queryJson)
result = self.client.search(index="hex_color_list", body=query)
with open("./resultJson/test.json", "w", encoding="utf-8") as f:
json.dump(result, fp=f, ensure_ascii=False, indent=3)
f.close()
queryJson.close()
"""
"""
def defualTermQuery(self):
queryJson = open("./ddl/term-query.json", "r")
query = json.load(queryJson)
result = self.client.search(index="hex_color_list", body=query)
with open("./resultJson/termQueryResult.json", "w", encoding="utf-8") as f:
json.dump(result, fp=f, ensure_ascii=False, indent=3)
f.close()
queryJson.close()
def defualMatchQuery(self):
queryJson = open("./ddl/match-query.json", "r")
query = json.load(queryJson)
result = self.client.search(index="hex_color_list", body=query)
with open("./resultJson/matchQueryResult.json", "w", encoding="utf-8") as f:
json.dump(result, fp=f, ensure_ascii=False, indent=3)
f.close()
queryJson.close()
@classmethod
def yamlFileRead(cls):
try:
f = open("./element/info.yaml", "r")
except FileNotFoundError as e:
print (e)
f.close()
exit(1)
else:
ip = dict(load(f.read())).get("ip")
f.close()
return ip
def main():
elnode = Elaserv()
# elnode.defaultMatchAll()
# elnode.defualTermQuery()
elnode.defualMatchQuery()
if __name__ == "__main__":
main()
"""
curl -X GET 'localhost:9200/hex_color_list/_search?pretty' -H 'Content-Type: application/json' -d '
{
"query": {
"match_all": {}
}
}
'
"""
{
"query": {
"match": {
"name": "add8e6"
}
}
}'ELK > elasticsearch' 카테고리의 다른 글
| Elasticsearch basic01 (0) | 2019.04.02 |
|---|---|
| elastisearch java api (0) | 2019.03.26 |
| 공부중 (0) | 2019.03.18 |
| java api - nestedQuery (0) | 2019.03.12 |
| java elasticsearch ( 주말에 정리할 코드 ) (0) | 2019.03.08 |
공부중
엘라스틱 서치 정리.docx'ELK > elasticsearch' 카테고리의 다른 글
| elastisearch java api (0) | 2019.03.26 |
|---|---|
| elasticsearch python 주말에 정리할 것 (0) | 2019.03.18 |
| java api - nestedQuery (0) | 2019.03.12 |
| java elasticsearch ( 주말에 정리할 코드 ) (0) | 2019.03.08 |
| python + elasticsearch (0) | 2019.02.21 |
java api - nestedQuery
static public void tstFunc() {
//GetIndexRequest request = new GetIndexRequest()
SearchRequest sr = new SearchRequest();
SearchSourceBuilder sb = new SearchSourceBuilder();
sb.query(QueryBuilders.nestedQuery
("obj1", QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("obj1.name", "kimjh"))
.must(QueryBuilders.rangeQuery("obj1.count")
.from(5)
.to(11)
.includeLower(true)
.includeUpper(true)), ScoreMode.Avg));
sr.source(sb);
try {
SearchResponse response = rhlc.search(sr, RequestOptions.DEFAULT);
SearchHits searchHits = response.getHits();
for (SearchHit hit : searchHits) {
// totalHits
System.out.println(hit.getSourceAsString());
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
'ELK > elasticsearch' 카테고리의 다른 글
| elasticsearch python 주말에 정리할 것 (0) | 2019.03.18 |
|---|---|
| 공부중 (0) | 2019.03.18 |
| java elasticsearch ( 주말에 정리할 코드 ) (0) | 2019.03.08 |
| python + elasticsearch (0) | 2019.02.21 |
| python + elasticsearch + 현재 정리 중 (0) | 2019.02.19 |
java elasticsearch ( 주말에 정리할 코드 )
package com.Elastic.srv;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
public class ElasticSearchConnector {
static private RestHighLevelClient _client = null;
private ElasticSearchConnector() {
}
static public RestHighLevelClient getClient() {
if (_client == null) {
_client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("192.168.17.136", 9200, "http")));
}
return _client;
}
}
package com.Elastic.srv;
import java.io.IOException;
import org.apache.lucene.queryparser.flexible.core.builders.QueryBuilder;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
public class MAIN {
static private String defaultIndex = "hex_color_list"; // index
static private RestHighLevelClient rhlc = null;
static public void main(String[] args) throws IOException {
System.out.println("test ... 중");
rhlc = ElasticSearchConnector.getClient();
doMatchingQuery();
//defaultFullTextSearch();
System.out.println("test ... 끝");
}
static public void doMatchingQuery() throws IOException {
// [1] create SearchRequest
SearchRequest sr = new SearchRequest();
SearchSourceBuilder sb = new SearchSourceBuilder();
sb.query(QueryBuilders.matchQuery("name", "b84c4c"));
sr.indices(defaultIndex);
sr.source(sb);
SearchResponse response = rhlc.search(sr, RequestOptions.DEFAULT);
SearchHits searchHits = response.getHits();
//System.out.println(searchHits);
// 매칭 되는 갯수
long totalHits = searchHits.getTotalHits();
// 얼마만큼 매칭 되는지
double maxScore = searchHits.getMaxScore();
System.out.println(totalHits);
System.out.println(maxScore);
for (SearchHit hit : searchHits) {
// totalHits
System.out.println(hit.getSourceAsString());
}
}
static public void defaultFullTextSearch() throws IOException {
// [1] create SearchRequest
SearchRequest sr = new SearchRequest();
SearchSourceBuilder sb = new SearchSourceBuilder();
sb.query(QueryBuilders.matchAllQuery());
sr.indices(defaultIndex);
sr.source(sb);
SearchResponse response = rhlc.search(sr, RequestOptions.DEFAULT);
SearchHit[] searchHits = response.getHits().getHits();
//System.out.println(searchHits);
for (SearchHit hit : searchHits) {
System.out.println(hit.getSourceAsString());
}
}
}
'ELK > elasticsearch' 카테고리의 다른 글
| 공부중 (0) | 2019.03.18 |
|---|---|
| java api - nestedQuery (0) | 2019.03.12 |
| python + elasticsearch (0) | 2019.02.21 |
| python + elasticsearch + 현재 정리 중 (0) | 2019.02.19 |
| python + elasticsearch (0) | 2019.02.16 |
python + elasticsearch
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
import openpyxl
import json
class ProjIndex:
def __init__(self):
self.esnode = Elasticsearch(hosts="192.168.2.10")
self.category = ["TU", "MC", "CH", "ET", "FO", "KE", "KM"]
def searchIndx(self):
with open("./index_list.json", "r", encoding="utf-8") as f:
jdoc = json.loads(f.read())
for c in self.category:
for v in dict(jdoc[c]).keys():
print ("{cat} #[{indx_name}] => {indx_cnt}".format(
cat = c,
indx_name = v,
indx_cnt = Search(using=self.esnode).index(jdoc[c][v]).count()))
def main():
pnode = ProjIndex()
pnode.searchIndx()
if __name__ == "__main__":
main()
'ELK > elasticsearch' 카테고리의 다른 글
| java api - nestedQuery (0) | 2019.03.12 |
|---|---|
| java elasticsearch ( 주말에 정리할 코드 ) (0) | 2019.03.08 |
| python + elasticsearch + 현재 정리 중 (0) | 2019.02.19 |
| python + elasticsearch (0) | 2019.02.16 |
| python + elasticsearch + 조회/삽입/생성 (0) | 2019.02.03 |
python + elasticsearch + 현재 정리 중
from elasticsearch import Elasticsearch
import json
class E:
def __init__(self):
self.esnode = Elasticsearch(hosts="192.168.2.10")
def totalIndex(self):
# 전체 인덱스 조회
pass
def searchCnt(self):
c = self.esnode.count(index="tb_frip_tourareacode", doc_type="doc")
print (c["count"])
# docs = self.esnode.search(index="tb_frip_tourareacode",
# doc_type="doc",
# body={
# "query": {
# "match_all":{
#
# }
# }
# })
#
# print (json.dumps(docs, indent=2, ensure_ascii=False))
def main():
node = E()
node.searchCnt()
if __name__ == "__main__":
main()
'ELK > elasticsearch' 카테고리의 다른 글
| java elasticsearch ( 주말에 정리할 코드 ) (0) | 2019.03.08 |
|---|---|
| python + elasticsearch (0) | 2019.02.21 |
| python + elasticsearch (0) | 2019.02.16 |
| python + elasticsearch + 조회/삽입/생성 (0) | 2019.02.03 |
| search java api (0) | 2019.01.29 |
python + elasticsearch
'''
문화재 검색 목록/ 문화재 상세 검색
@ 작성자 : 김준현
@ 작성일 :
@ url : https://www.cha.go.kr/html/HtmlPage.do?pg=/publicinfo/pbinfo3_0202.jsp&mn=NS_04_04_02
'''
from urllib.parse import urlencode
import requests
import os
import time
import sys
from COMM.UrlInit import UrlInit
from COMM.DocParcing import DocParcing
class ClastList:
def __init__(self):
self.reqUrlList = UrlInit.urlReturn(cat1="CH", cat2="ClastList", cat3="url")
self.reqUrlDetail = UrlInit.urlReturn(cat1="CH", cat2="ClastDetail", cat3="url")
self.params = {"pageUnit":100, "pageIndex":0}
self.clastListElements = [] # 문화재 검색 목록
self.clastDetailElements = [] # 문화재 상세 검색 목록
def doRequestsList(self):
fetchCount = 0
while True:
try:
fetchCount += self.params['pageUnit']
self.params["pageIndex"] += 1
time.sleep(1)
html = requests.get(self.reqUrlList + UrlInit.urlEncoding(self.params))
except requests.ConnectionError as e:
print ("error message : " + e)
sys.exit(1)
else:
if str(html.status_code)[0] == "2":
xmlTree = DocParcing.respoToXML(html.text)
totalCnt = int(xmlTree.find("totalCnt").text, base=10)
print (totalCnt, fetchCount)
for c in xmlTree.findall('item'):
element = {"ccmaName": c.find("ccmaName").text,
"ccbaMnm1": c.find("ccbaMnm1").text,
"ccbaMnm2": c.find("ccbaMnm2").text,
"ccbaCtcdNm":c.find("ccbaCtcdNm").text,
"ccsiName": c.find("ccsiName").text,
"ccbaAdmin": c.find("ccbaAdmin").text,
"ccbaKdcd": c.find("ccbaKdcd").text, #문화재검색 상세
"ccbaCtcd": c.find("ccbaCtcd").text, #문화재검색 상세
"ccbaAsno": c.find("ccbaAsno").text, #문화재검색 상세
"ccbaCncl": c.find("ccbaCncl").text,
"ccbaCpno": c.find("ccbaCpno").text,
"longitude": c.find("longitude").text,
"latitude": c.find("latitude").text,
"cllcttime": time.strftime("%Y%m%d")}
if None not in element.values():
self.clastListElements.append(element)
print (element)
else:
# 응답 코드가 : 3xx, 4xx, 5xx 인 경우
break
if fetchCount > totalCnt:
break
# self.doRequestsDetail()
DocParcing.doJsonFile(html_doc=self.clastListElements, json_file_path="./JSON/ClastList.json")
# DocParcing.doJsonFile(html_doc=self.clastDetailElements, json_file_path="./JSON/ClastDetail.json")
def doRequestsDetail(self):
for i in self.clastListElements:
tmp = {"ccbaKdcd" : i["ccbaKdcd"], "ccbaCtcd" : i["ccbaCtcd"], "ccbaAsno" : i["ccbaAsno"]}
try:
time.sleep(0.5)
html = requests.get(self.reqUrlDetail + UrlInit.urlEncoding(tmp))
except requests.ConnectionError as e:
print ("error message : " + e)
sys.exit(1)
else:
if str(html.status_code)[0] == "2":
xmlTree = DocParcing.respoToXML(html.text)
for c in xmlTree.findall('item'):
element = {"ccmaName": c.find("ccmaName").text,
"ccbaMnm1": c.find("ccbaMnm1").text,
"ccbaMnm2": c.find("ccbaMnm2").text,
"gcodeName": c.find("gcodeName").text,
"bcodeName": c.find("bcodeName").text,
"mcodeName": c.find("mcodeName").text,
"scodeName": c.find("scodeName").text,
"ccbaQuan": c.find("ccbaQuan").text,
"ccbaAsdt": c.find("ccbaAsdt").text,
"ccbaCtcdNm": c.find("ccbaCtcdNm").text,
"ccsiName": c.find("ccsiName").text,
"ccceName": c.find("ccceName").text,
"ccbaPoss": c.find("ccbaPoss").text,
"ccbaAdmin": c.find("ccbaAdmin").text,
"ccbaCncl": c.find("ccbaCncl").text,
"ccbaCndt": c.find("ccbaCndt").text,
"imageUrl": c.find("imageUrl").text,
"content": c.find("content").text,}
print (element)
self.clastDetailElements.append(element)
def main():
clastList = ClastList()
clastList.doRequestsList()
if __name__ == "__main__":
main()
======================================================================================================
from elasticsearch import Elasticsearch, helpers
import json
from COMM import Elastic
class ClastListElastic:
def __init__(self):
self.es = Elastic.Elastic.srvReturn()
def indexCreateList(self):
# 목화재 검색 목록
self.es.indices.create(
index="tb_frip_clastlist",
body= {
"settings": {
"number_of_shards": 5
},
"filter": {
"mutate": {
"remove_field":["@version", "@timestamp"]
}
},
"mappings":{
"doc":{
"properties":{
"ccmaName" : {"type": "text"},
"ccbaMnm1" : {"type": "text"},
"ccbaMnm2" : {"type": "text"},
"ccbaCtcdNm": {"type": "text"},
"ccsiName" : {"type": "text"},
"ccbaAdmin" : {"type": "text"},
"ccbaKdcd" : {"type": "text"},
"ccbaCtcd" : {"type": "text"},
"ccbaAsno" : {"type": "text"},
"ccbaCncl" : {"type": "text"},
"ccbaCpno" : {"type": "text"},
"longitude" : {"type": "text"},
"latitude" : {"type": "text"},
"cllcttime" : {"type": "text"},
}
}
}
}
)
def indexCreateDetail(self):
# 문화재 상세목록
self.es.indices.create(
index="tb_frip_clastDetail",
body={
"settings": {
"number_of_shards": 5
},
"filter": {
"mutate": {
"remove_field": ["@version", "@timestamp"]
}
},
"mappings": {
"doc": {
"properties": {
"ccmaName" : {"type": "text"},
"ccbaMnm1" : {"type": "text"},
"ccbaMnm2" : {"type": "text"},
"gcodeName" : {"type": "text"},
"bcodeName" : {"type": "text"},
"mcodeName" : {"type": "text"},
"scodeName" : {"type": "text"},
"ccbaQuan" : {"type": "text"},
"ccbaAsdt" : {"type": "text"},
"ccbaCtcdNm": {"type": "text"},
"ccsiName" : {"type": "text"},
"ccceName" : {"type": "text"},
"ccbaPoss" : {"type": "text"},
"ccbaAdmin" : {"type": "text"},
"ccbaCncl" : {"type": "text"},
"ccbaCndt" : {"type": "text"},
"imageUrl" : {"type": "text"},
"content" : {"type": "text"},
}
}
}
}
)
def jsonFileInsertList(self):
# json file 데이터 적재
with open("./JSON/ClastList.json", "r", encoding="utf-8") as jsonfile:
doc = json.load(jsonfile)
jsonfile.close()
for n, p in enumerate(doc):
print (p)
self.es.index(index="tb_frip_clastlist", doc_type="doc", id=n+1, body=p)
'ELK > elasticsearch' 카테고리의 다른 글
| python + elasticsearch (0) | 2019.02.21 |
|---|---|
| python + elasticsearch + 현재 정리 중 (0) | 2019.02.19 |
| python + elasticsearch + 조회/삽입/생성 (0) | 2019.02.03 |
| search java api (0) | 2019.01.29 |
| python + elasticsearch api (0) | 2019.01.29 |
python + elasticsearch + 조회/삽입/생성
from elasticsearch import Elasticsearch
import pprint as ppr
import json
class ElaAPI:
es = Elasticsearch(hosts="192.168.240.129", port=9200) # 객체 생성
@classmethod
def srvHealthCheck(cls):
health = cls.es.cluster.health()
print (health)
@classmethod
def allIndex(cls):
# Elasticsearch에 있는 모든 Index 조회
print (cls.es.cat.indices())
@classmethod
def dataInsert(cls):
# ===============
# 데이터 삽입
# ===============
with open("../json_doc_make/tst.json", "r", encoding="utf-8") as fjson:
data = json.loads(fjson.read())
for n, i in enumerate(data):
doc = {"cont" :i['cont'],
"mnagnnm":i["mnagnnm"],
"post" :i["post"],
"rgdt" :i["rgdt"],
"rgter" :i["rgter"],
"tel" :i["tel"],
"title" :i["title"]}
res = cls.es.index(index="today19020301", doc_type="today", id=n+1, body=doc)
print (res)
@classmethod
def searchAll(cls, indx=None):
# ===============
# 데이터 조회 [전체]
# ===============
res = cls.es.search(
index = "today19020301", doc_type = "today",
body = {
"query":{"match_all":{}}
}
)
print (json.dumps(res, ensure_ascii=False, indent=4))
@classmethod
def searchFilter(cls):
# ===============
# 데이터 조회 []
# ===============
res = cls.es.search(
index = "today19020301", doc_type = "today",
body = {
"query": {"match":{"post":"산림교육문화과"}}
}
)
ppr.pprint(res)
@classmethod
def createIndex(cls):
# ===============
# 인덱스 생성
# ===============
cls.es.indices.create(
index = "today19020301",
body = {
"settings": {
"number_of_shards": 5
},
"mappings": {
"today":{
"properties": {
"cont": {"type": "text"},
"mnagnnm": {"type": "text"},
"post": {"type": "text"},
"rgdt": {"type": "text"},
"rgter": {"type": "text"},
"tel": {"type": "text"},
"title": {"type": "text"}
}
}
}
}
)
ElaAPI.allIndex()
# ElaAPI.srvHealthCheck()
# ElaAPI.createIndex()
# ElaAPI.dataInsert()
# ElaAPI.searchAll()
# ElaAPI.searchFilter()
'ELK > elasticsearch' 카테고리의 다른 글
| python + elasticsearch + 현재 정리 중 (0) | 2019.02.19 |
|---|---|
| python + elasticsearch (0) | 2019.02.16 |
| search java api (0) | 2019.01.29 |
| python + elasticsearch api (0) | 2019.01.29 |
| java api elasticsearch aggregation (0) | 2019.01.28 |
search java api
package com.my.search;
import java.util.HashMap;
import java.util.Map;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.script.mustache.SearchTemplateRequestBuilder;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import com.my.srv.Server;
/*
*
* @ 작성일 : 2019-01-29
* @ 주제 : Search
* @ 작성자 : JunHyeon Kim
* @ 이메일 : junhyeon.kim@ezfarm.co.kr
*/
public class ElasticSearch extends Server {
// ###########################################
SearchResponse response = null;
JSONParser parser = new JSONParser();
JSONObject json = null;
JSONArray jsonArry = null;
// ###########################################
public void doSearch() {
// server
Server.ServerSetting();
Map<String, Object> template_params = new HashMap<String, Object>();
template_params.put("local", "경북");
// Search Test
response = new SearchTemplateRequestBuilder(client)
.setScript("{\n" +
" \"query\" : {\n" +
" \"match\" : {\n" +
" \"area\" : \"{{local}}\"\n" +
" }\n" +
" }\n" +
"}")
.setScriptType(ScriptType.INLINE)
.setScriptParams(template_params)
.setRequest(new SearchRequest("tb_frip_dayshow"))
.get()
.getResponse();
}
public void doJson() {
// Json file 로 변환 ==============================
JSONObject jsnObj = null;
JSONArray jsnArr = new JSONArray();
// ==============================================
try {
json = (JSONObject) parser.parse(response.toString());
json = (JSONObject) json.get("hits");
jsonArry = (JSONArray) json.get("hits");
//System.out.println(jsonArry);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
for (int i = 0; i < jsonArry.size(); i++) {
JSONObject tmpnode = (JSONObject)jsonArry.get(i);
tmpnode = (JSONObject)tmpnode.get("_source");
jsnObj = new JSONObject(); // 객체 생성
jsnObj.put("title", tmpnode.get("title"));
jsnObj.put("place", tmpnode.get("place"));
jsnObj.put("startDate", tmpnode.get("startDate"));
jsnObj.put("gpsX", tmpnode.get("gpsX"));
jsnObj.put("gpsY", tmpnode.get("gpsY"));
jsnArr.add(jsnObj);
}
System.out.println(jsnArr);
}
}
'ELK > elasticsearch' 카테고리의 다른 글
| python + elasticsearch (0) | 2019.02.16 |
|---|---|
| python + elasticsearch + 조회/삽입/생성 (0) | 2019.02.03 |
| python + elasticsearch api (0) | 2019.01.29 |
| java api elasticsearch aggregation (0) | 2019.01.28 |
| java api2 (0) | 2019.01.25 |
python + elasticsearch api
from elasticsearch import Elasticsearch, exceptions
class Elas:
def __init__(self):
self.esnode = None
def ElasticServSetting(self):
try:
self.esnode = Elasticsearch(hosts="192.168.240.129", port="9200")
except exceptions.ConnectionError as e:
print (e)
else:
settings= {
"settings": {
"number_of_shards": 3
},
"mappings": {
"doc": {
"properties": {
"numb" : {"type": "integer"},
"name" : {"type": "text"}
}
}
}
}
# create index
self.esnode.indices.create(index="indx190128", ignore=400, body=settings)
def main():
node = Elas() # 객체 생성
node.ElasticServSetting()l
if __name__ == "__main__":
main()
'ELK > elasticsearch' 카테고리의 다른 글
| python + elasticsearch + 조회/삽입/생성 (0) | 2019.02.03 |
|---|---|
| search java api (0) | 2019.01.29 |
| java api elasticsearch aggregation (0) | 2019.01.28 |
| java api2 (0) | 2019.01.25 |
| java api (0) | 2019.01.25 |
