nginx setting
server {
listen 560
location / {
proxy_set_header HOST $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Authorization "Basic ZWxhc3RpYzplbGFzdGlj"
proxy_pass http://localhost:5601
}
}
'ELK > elasticsearch' 카테고리의 다른 글
대상 인덱스의 field 모두를 fielddata true로 변환하는 방법 (0) | 2020.01.17 |
---|---|
app-search (0) | 2020.01.06 |
python-appsearch (0) | 2019.12.19 |
Elasticsearch + python + pipeline (0) | 2019.12.02 |
네이버 무비 정보 크롤링 => logstash data pipe_line => kibana 시각화 (0) | 2019.11.10 |
python-appsearch
from swiftype_app_search import Client
class AppSrch():
def __init__(self):
self._clientNode = Client(
api_key="i",
base_endpoint="172.16.40.139:3002/api/as/v1",
use_https=False
)
self._engineName = "kimjunhyeon"
def dataInsert(self):
document = {
"id" : "doc_"
,"txt" : "aGVsbG8gd29ybGQ="
,"page": 2
}
#self._clientNode.index_document(engine_name=self._engineName, document=document)
self._clientNode.update_documents(self._engineName, document)
if __name__ == "__main__":
obj = AppSrch()
obj.dataInsert()
'ELK > elasticsearch' 카테고리의 다른 글
app-search (0) | 2020.01.06 |
---|---|
nginx setting (0) | 2019.12.19 |
Elasticsearch + python + pipeline (0) | 2019.12.02 |
네이버 무비 정보 크롤링 => logstash data pipe_line => kibana 시각화 (0) | 2019.11.10 |
python + elasticsearch : csv => bulk json 변환 (0) | 2019.10.23 |
Elasticsearch + python + pipeline
from ELa import Ela
class Client:
def __init__(self):
self._elaClinet = Ela.retElanode()
self._targetDir = ""
def insrtDoc(self):
d = {"data":"aGVsbG8="}
self._elaClinet.index(index="x_index", pipeline="attachment", body=d)
o = Client()
o.insrtDoc()
'ELK > elasticsearch' 카테고리의 다른 글
nginx setting (0) | 2019.12.19 |
---|---|
python-appsearch (0) | 2019.12.19 |
네이버 무비 정보 크롤링 => logstash data pipe_line => kibana 시각화 (0) | 2019.11.10 |
python + elasticsearch : csv => bulk json 변환 (0) | 2019.10.23 |
elasticsearch SSL 적용 connect code + python (0) | 2019.10.22 |
네이버 무비 정보 크롤링 => logstash data pipe_line => kibana 시각화
input {
stdin { }
}
filter {
json {
source => "message"
}
mutate {
remove_field => ["@version", "host", "message", "path"]
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["http://192.168.42.136:9200"]
index => nvr_movie
}
}
logstash 코드
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlencode
import json
from Utils import Utils
#
# author : JunHyeon.Kim
# date : 20191110
# ver : 0.1
# naver move data cwling
#
#
class MV:
def __init__(self):
INFO_ = Utils.getSetting()
self._cllTime = Utils.getCllTime()
self._category = Utils.getCategory()
self._totalData = list()
self._urlInfo = {"url" : INFO_["url"], "path": INFO_["path"], "time": self._cllTime}
def urlRequests(self):
for i in self._category["category"]:
urlArgs_ = urlencode(
{
"sel" : "cnt",
"date": self._urlInfo["time"],
"tg" : i
}
)
print (urlArgs_, self._category["category"][i])
requestUrl_ = self._urlInfo["url"] + self._urlInfo["path"] +"?"+ urlArgs_
try:
htmlObj = requests.get(requestUrl_)
except requests.exceptions.ConnectionError as E:
print (E)
exit(1)
else:
if htmlObj.status_code == 200:
bsObj = BeautifulSoup(htmlObj.text, "html.parser")
titles = bsObj.select("td.title > div.tit3 > a")
with open("./nvr_movie_"+ self._cllTime +"_.json", "a", encoding="utf-8") as f:
for c, t in enumerate(titles):
d = {
"title" : t.attrs["title"],
"rank" : c+1,
"clltime": self._cllTime,
"genr" : self._category["category"][i] }
f.write(json.dumps(d ,ensure_ascii=False) + "\n")
f.close()
def main():
mvObj = MV()
mvObj.urlRequests()
if __name__ == "__main__":
main()

'ELK > elasticsearch' 카테고리의 다른 글
python-appsearch (0) | 2019.12.19 |
---|---|
Elasticsearch + python + pipeline (0) | 2019.12.02 |
python + elasticsearch : csv => bulk json 변환 (0) | 2019.10.23 |
elasticsearch SSL 적용 connect code + python (0) | 2019.10.22 |
logstash_01 / json (0) | 2019.10.19 |
python + elasticsearch : csv => bulk json 변환
## csv => bulk type의 파일로 변환
def csvFileRead(self):
with open(self._csv_path, "r", encoding="utf-8") as csv_file:
with open("/home/elastic/test.dir/py_test/the_planet_code/the_planet.json", "a", encoding="utf-8") as f:
csv_reader = csv.DictReader(csv_file, delimiter=",")
for c, r in enumerate(csv_reader):
h = json.dumps({"index": {"_index": "the_planet_poc_index_1", "_type": "_doc", "_id": "the_planet_" + str(c+1)}},
ensure_ascii=False)
f.write(h + "\n")
e = json.dumps((dict(r)),
ensure_ascii=False)
f.write(e + "\n")
time.sleep(2)
f.close()
csv_file.close()
'ELK > elasticsearch' 카테고리의 다른 글
Elasticsearch + python + pipeline (0) | 2019.12.02 |
---|---|
네이버 무비 정보 크롤링 => logstash data pipe_line => kibana 시각화 (0) | 2019.11.10 |
elasticsearch SSL 적용 connect code + python (0) | 2019.10.22 |
logstash_01 / json (0) | 2019.10.19 |
elasticsearch + java _api + match_all (0) | 2019.06.04 |
elasticsearch SSL 적용 connect code + python
@classmethod
def getElaNode(cls):
try:
f = open("/home/elastic/test.dir/py_test/conf.yml", "r", encoding="utf-8")
except FileNotFoundError as E:
print (E)
exit(1)
else:
INF = yaml.load(f, yaml.Loader)
retNode = Elasticsearch(
host = INF.get("ela_host"),
port = 9200,
http_auth = (INF.get("user"),
INF.get("psswd"))
)
if retNode.ping():
#print (retNode.info())
return retNode
'ELK > elasticsearch' 카테고리의 다른 글
네이버 무비 정보 크롤링 => logstash data pipe_line => kibana 시각화 (0) | 2019.11.10 |
---|---|
python + elasticsearch : csv => bulk json 변환 (0) | 2019.10.23 |
logstash_01 / json (0) | 2019.10.19 |
elasticsearch + java _api + match_all (0) | 2019.06.04 |
elasticsearch : count (0) | 2019.05.21 |
logstash_01 / json
input {
stdin {}
}
filter {
json {
source => "message"
}
mutate {
remove_field => ["path", "@version", "message", "host", "@timestamp"]
}
}
output {
stdout {
codec => rubydebug { }
}
elasticsearch {
hosts => "192.168.240.183:9200"
index => "kim"
document_type => "_doc"
}
}
'ELK > elasticsearch' 카테고리의 다른 글
python + elasticsearch : csv => bulk json 변환 (0) | 2019.10.23 |
---|---|
elasticsearch SSL 적용 connect code + python (0) | 2019.10.22 |
elasticsearch + java _api + match_all (0) | 2019.06.04 |
elasticsearch : count (0) | 2019.05.21 |
elasticsearch + shell01 (0) | 2019.05.11 |
elasticsearch + java _api + match_all
package com.jh.ela;
import java.io.IOException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import static org.elasticsearch.index.query.QueryBuilders.*;
/**
* Builder pattern
* @author ezfarm
*
*/
import com.jh.server.Server;
public class Srcher {
String srchwrd ; // 검색어
Server srvernode;
RestHighLevelClient elanode;
String searchIndex;
int size;
int from;
static public class Builder {
String srchwrd = null;
Server srvernode = null;
RestHighLevelClient elanode = null;
String searchIndex = null;
int size = 0;
int from = 0;
public Builder(String srchwrd) {
// TODO Auto-generated constructor stub
// 검색어
this.srchwrd = srchwrd;
}
/**
* 페이지 설정
* @param size, from
* @return
*/
public Builder withPages (int size, int from) {
// server
this.srvernode = new Server();
// 검색 대상 인덱스
this.searchIndex = this.srvernode.searchIndex;
this.size = size;
this.from = from;
return this;
}
public Srcher build() {
return new Srcher(this);
}
}
public Srcher(Builder builder) {
srchwrd = builder.srchwrd;
srvernode = builder.srvernode;
elanode = builder.srvernode.getElaserver();
searchIndex = builder.searchIndex;
}
/**
* Client node close !!!
*/
public void serverDie() {
try {
elanode.close();
System.out.println("client close !!!");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// 검색 테스트
public void fullTextSearch() {
/*
* "query" : {
* "size" : 180,
* "match_all": {}
* }
*/
System.out.println("searchIndex => " + searchIndex);
SearchRequest searchRequest = new SearchRequest(searchIndex);
SearchResponse searchResponse = new SearchResponse();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(matchAllQuery());
searchSourceBuilder.size(180);
searchRequest.source(searchSourceBuilder);
if (elanode != null) {
System.out.println("RestHighLevel node success!!!");
}
try {
searchResponse = elanode.search(searchRequest, RequestOptions.DEFAULT);
System.out.println("status => " + searchResponse.status());
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
System.out.println(hit.getSourceAsString());
}
} catch (ElasticsearchException e) {
System.out.println(e);
} catch (IOException e) {
// TODO Auto-generated catch block
System.out.println(e);
}
}
}
'ELK > elasticsearch' 카테고리의 다른 글
elasticsearch SSL 적용 connect code + python (0) | 2019.10.22 |
---|---|
logstash_01 / json (0) | 2019.10.19 |
elasticsearch : count (0) | 2019.05.21 |
elasticsearch + shell01 (0) | 2019.05.11 |
elastic 불용어 테스트 (0) | 2019.05.01 |
elasticsearch : count
#!/bin/bash
curl -XGET 'http://192.168.240.10:9200/movie-index/_count?pretty' -H 'Content-Type: application/json' -d '
{
"query" : {
"term" : { "movie_jangre" : "드라마" }
}
}'
'ELK > elasticsearch' 카테고리의 다른 글
logstash_01 / json (0) | 2019.10.19 |
---|---|
elasticsearch + java _api + match_all (0) | 2019.06.04 |
elasticsearch + shell01 (0) | 2019.05.11 |
elastic 불용어 테스트 (0) | 2019.05.01 |
python + bulk + insert + code (0) | 2019.04.24 |
elasticsearch + shell01
#!/bin/bash
index_name=$1
if [ "$index_name" != "" ];then
curl -X GET "http://192.168.240.10:9200/{$index_name}/_count?pretty"
fi
실행
$ ./index-count.sh kimjh
{
"count" : 17,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
}
}
#!/bin/bash
echo "==================================================="
echo " (주) "
echo " 날짜 : `date "+%Y-%m-%d"`"
echo " 작성자 : 김준현/ 매니져"
echo "==================================================="
response=$(curl -XGET 'http://192.168.240.10:9200/_cat/indices/test*?pretty')
echo $response
echo "==================================================="
'ELK > elasticsearch' 카테고리의 다른 글
elasticsearch + java _api + match_all (0) | 2019.06.04 |
---|---|
elasticsearch : count (0) | 2019.05.21 |
elastic 불용어 테스트 (0) | 2019.05.01 |
python + bulk + insert + code (0) | 2019.04.24 |
elastic bacis#3 (0) | 2019.04.23 |