python-appsearch

ELK/elasticsearch2019. 12. 19. 09:54

from swiftype_app_search import Client

class AppSrch():


    def __init__(self):

        self._clientNode = Client(
            api_key="i",
            base_endpoint="172.16.40.139:3002/api/as/v1",
            use_https=False
        )

        self._engineName = "kimjunhyeon"

    def dataInsert(self):


        document = {
            "id"   : "doc_"
            ,"txt" : "aGVsbG8gd29ybGQ="
            ,"page": 2
        }

        #self._clientNode.index_document(engine_name=self._engineName, document=document)
        self._clientNode.update_documents(self._engineName, document)


if __name__ == "__main__":
    obj = AppSrch()
    obj.dataInsert()

logstash + app-search

ELK/logstash2019. 12. 10. 14:55

input {
    file {
        start_position => "beginning"
        path           => ["/opt/logstash-7.2.0/conf/test.json"]
        sincedb_path   => "/dev/null"
        codec          => "json"
    }
}

filter {
    mutate {
        remove_field => ["path", "@version", "@timestamp", "host"]
    }
}

output {
    stdout {}
    elastic_app_search {
        api_key => "private-fobwwhsirra3cgner79khnhi"
        engine  => "kimjunhyeon"
        url     => "http://172.16.40.139:3002"
        path    => "/api/as/v1/"
        id      => "test_01"
    }
}

'ELK > logstash' 카테고리의 다른 글

logstash file json absolute path  (0) 2020.06.17
logstash input-plugin (elasticsearch)  (0) 2020.05.12
logstash plugin install  (0) 2020.04.15
logstash + python  (0) 2019.01.12
logstash - mysql - elasticsearch 연동  (0) 2019.01.06

from ELa import Ela

class Client:


    def __init__(self):

        self._elaClinet = Ela.retElanode()
        self._targetDir = ""

    def insrtDoc(self):

        d = {"data":"aGVsbG8="}

        self._elaClinet.index(index="x_index", pipeline="attachment", body=d)

o = Client()
o.insrtDoc()

 

input {
    stdin { }
}

filter {
    json {
        source => "message"
    }
    mutate {
        remove_field => ["@version", "host", "message", "path"]
    }
}

output {
    stdout { codec => rubydebug }
    elasticsearch {
        hosts => ["http://192.168.42.136:9200"]
        index => nvr_movie
    }
}

logstash 코드 

 

 

import requests
from bs4 import BeautifulSoup
from urllib.parse import urlencode
import json 

from Utils import Utils

#
# author : JunHyeon.Kim
# date   : 20191110
# ver    : 0.1
# naver move data cwling
#
#

class MV:

    def __init__(self):
        
        INFO_           = Utils.getSetting()  
        self._cllTime   = Utils.getCllTime()
        self._category  = Utils.getCategory() 
        self._totalData = list()         
        self._urlInfo   = {"url" : INFO_["url"], "path": INFO_["path"], "time": self._cllTime}

    def urlRequests(self):
        
        for i in self._category["category"]:
                         
            urlArgs_ = urlencode(
                {
                    "sel" : "cnt", 
                    "date": self._urlInfo["time"], 
                    "tg"  : i 
                }
            )

            print (urlArgs_,  self._category["category"][i])

            requestUrl_ = self._urlInfo["url"] + self._urlInfo["path"] +"?"+ urlArgs_ 
                
            try:

                htmlObj = requests.get(requestUrl_)
            except requests.exceptions.ConnectionError as E:
                print (E)
                exit(1)
            else:
                
                if htmlObj.status_code == 200:

                    bsObj  = BeautifulSoup(htmlObj.text, "html.parser")

                    titles = bsObj.select("td.title > div.tit3 > a")
                
                    with open("./nvr_movie_"+ self._cllTime +"_.json", "a", encoding="utf-8") as f:
                        
                        for c, t in enumerate(titles):
                            
                            d = {
                                "title"  : t.attrs["title"], 
                                "rank"   : c+1,
                                "clltime": self._cllTime, 
                                "genr"   : self._category["category"][i] } 

                            f.write(json.dumps(d ,ensure_ascii=False) + "\n")

                        f.close()
                        
def main():

    mvObj = MV()
    mvObj.urlRequests() 

if __name__ == "__main__":
    main()
        

'ELK > elasticsearch' 카테고리의 다른 글

python-appsearch  (0) 2019.12.19
Elasticsearch + python + pipeline  (0) 2019.12.02
python + elasticsearch : csv => bulk json 변환  (0) 2019.10.23
elasticsearch SSL 적용 connect code + python  (0) 2019.10.22
logstash_01 / json  (0) 2019.10.19

    ## csv => bulk type의 파일로 변환
    def csvFileRead(self):

        with open(self._csv_path, "r", encoding="utf-8") as csv_file:

            with open("/home/elastic/test.dir/py_test/the_planet_code/the_planet.json", "a", encoding="utf-8") as f:

                csv_reader = csv.DictReader(csv_file, delimiter=",")

                for c, r in enumerate(csv_reader):

                    h = json.dumps({"index": {"_index": "the_planet_poc_index_1", "_type": "_doc", "_id": "the_planet_" + str(c+1)}},
                                  ensure_ascii=False)
                    f.write(h + "\n")

                    e = json.dumps((dict(r)),
                                   ensure_ascii=False)
                    f.write(e + "\n")
                    time.sleep(2)

                f.close()
            csv_file.close()

    @classmethod
    def getElaNode(cls):

        try:

            f = open("/home/elastic/test.dir/py_test/conf.yml", "r", encoding="utf-8")
        except FileNotFoundError as E:
            print (E)
            exit(1)
        else:
            INF = yaml.load(f, yaml.Loader)

            retNode = Elasticsearch(
                    host      = INF.get("ela_host"),
                    port      = 9200,
                    http_auth = (INF.get("user"),
                                 INF.get("psswd"))
                    )

            if retNode.ping():
                #print (retNode.info())
                return retNode

input {
  stdin {}
}

filter {
  json {
    source => "message"
  }
  mutate {
    remove_field => ["path", "@version", "message", "host", "@timestamp"]
  }
}

output {
  stdout {
    codec => rubydebug { }
  }
  elasticsearch {
    hosts => "192.168.240.183:9200"
    index => "kim"
    document_type => "_doc"
  }
}

'ELK > elasticsearch' 카테고리의 다른 글

python + elasticsearch : csv => bulk json 변환  (0) 2019.10.23
elasticsearch SSL 적용 connect code + python  (0) 2019.10.22
elasticsearch + java _api + match_all  (0) 2019.06.04
elasticsearch : count  (0) 2019.05.21
elasticsearch + shell01  (0) 2019.05.11

package com.jh.ela;
import java.io.IOException;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import static org.elasticsearch.index.query.QueryBuilders.*;

/**
 * Builder pattern 
 * @author ezfarm
 *
 */
import com.jh.server.Server;

public class Srcher { 
	
	String srchwrd ;             // 검색어 
	Server srvernode;
	RestHighLevelClient elanode;
	String searchIndex;  
	int size;
	int from;
	
	static public class Builder {
		
		String srchwrd   = null;
		Server srvernode = null;
		RestHighLevelClient elanode = null;
		String searchIndex = null;
		int size = 0;
		int from = 0;
		
		public Builder(String srchwrd) {
			// TODO Auto-generated constructor stub
			// 검색어 
			this.srchwrd = srchwrd;
		}
		
		/**
		 * 페이지 설정 
		 * @param size, from 
		 * @return
		 */
		public Builder withPages (int size, int from) {
			
			// server 
			this.srvernode   = new Server();
			// 검색 대상 인덱스 
			this.searchIndex = this.srvernode.searchIndex;
			this.size = size;
			this.from = from;
			
			return this;
		}
		
		public Srcher build() {
			return new Srcher(this);
		}
	}
	
	public Srcher(Builder builder) {
		
		srchwrd     = builder.srchwrd;
		srvernode   = builder.srvernode;
		elanode     = builder.srvernode.getElaserver();
		searchIndex = builder.searchIndex;
		
	}
	
	/**
	 * Client node close !!!
	 */
	public void serverDie() {
		
		try {
			
			elanode.close();
			System.out.println("client close !!!");
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	// 검색 테스트
	public void fullTextSearch() {
		
		/*
		 *  "query" : {
		 *  	"size" : 180,
		 *  	"match_all": {}
		 *  }
		 */
		System.out.println("searchIndex =>  " + searchIndex);
		SearchRequest searchRequest = new SearchRequest(searchIndex);
		SearchResponse searchResponse = new SearchResponse();
		
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.query(matchAllQuery()); 
		searchSourceBuilder.size(180); 
		searchRequest.source(searchSourceBuilder);
		
		
		if (elanode != null) {
			System.out.println("RestHighLevel node success!!!");
		}
		
		try {
			
			searchResponse = elanode.search(searchRequest, RequestOptions.DEFAULT);
			System.out.println("status =>  " + searchResponse.status());
			SearchHits hits = searchResponse.getHits();
			
			for (SearchHit hit : hits) {
				System.out.println(hit.getSourceAsString());
			}
			
		} catch (ElasticsearchException e) {
			System.out.println(e);
			
		} catch (IOException e) {
			// TODO Auto-generated catch block
			System.out.println(e);
		} 
	}
}

'ELK > elasticsearch' 카테고리의 다른 글

elasticsearch SSL 적용 connect code + python  (0) 2019.10.22
logstash_01 / json  (0) 2019.10.19
elasticsearch : count  (0) 2019.05.21
elasticsearch + shell01  (0) 2019.05.11
elastic 불용어 테스트  (0) 2019.05.01

#!/bin/bash

curl -XGET 'http://192.168.240.10:9200/movie-index/_count?pretty' -H 'Content-Type: application/json' -d '
{
  "query" : {
    "term" : { "movie_jangre" : "드라마" }
  }
}'

'ELK > elasticsearch' 카테고리의 다른 글

logstash_01 / json  (0) 2019.10.19
elasticsearch + java _api + match_all  (0) 2019.06.04
elasticsearch + shell01  (0) 2019.05.11
elastic 불용어 테스트  (0) 2019.05.01
python + bulk + insert + code  (0) 2019.04.24

#!/bin/bash 

index_name=$1 

if [ "$index_name" != "" ];then 
  curl -X GET "http://192.168.240.10:9200/{$index_name}/_count?pretty" 
fi 

실행 

...더보기

$ ./index-count.sh kimjh
{
  "count" : 17,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "skipped" : 0,
    "failed" : 0
  }
}

#!/bin/bash

echo "==================================================="
echo " (주) "
echo " 날짜 : `date "+%Y-%m-%d"`"
echo " 작성자 : 김준현/ 매니져"
echo "==================================================="
response=$(curl -XGET 'http://192.168.240.10:9200/_cat/indices/test*?pretty')
echo $response
echo "==================================================="

 

'ELK > elasticsearch' 카테고리의 다른 글

elasticsearch + java _api + match_all  (0) 2019.06.04
elasticsearch : count  (0) 2019.05.21
elastic 불용어 테스트  (0) 2019.05.01
python + bulk + insert + code  (0) 2019.04.24
elastic bacis#3  (0) 2019.04.23