from csv to json convert + logstash
언어/python2019. 11. 26. 06:47
import csv
import json
import os
import shutil
import subprocess
import time
#
class ConvJson():
def __init__(self):
# 대상 디렉토리
self._targetPath = r""
# json 이동 디렉토리
self._jsonPath = "jsonDir"
# field_name
self._fieldName = ("a", "b", "c")
# logstash path
self._logstashRun = ""
self._targetConfDir = ""
def csvRead(self):
# 디렉토리 이동
os.chdir(self._targetPath)
# file list
fileList = os.listdir()
for f in fileList:
fileAbsPath = os.path.abspath(f)
fileName, fileExtension = os.path.splitext(fileAbsPath)
if fileExtension == ".csv":
try:
csvFile = open(fileAbsPath, "r", encoding="utf-8")
next(csvFile)
except FileNotFoundError as E:
print(E)
exit(1)
else:
""" ndjson 파일로 변환
"""
try:
jsonFile = open("{}.json".format(fileName), "w", encoding="utf-8")
except:
print("file error")
exit(1)
else:
reader = csv.DictReader(f=csvFile, fieldnames=self._fieldName, delimiter="|")
data = list(reader)
for x in range(0, len(data)):
# json.dump(data, fp=jsonFile)
if x != len(data)-1:
strData = json.dumps(data[x]) + "\n"
else:
strData = json.dumps(data[x])
jsonFile.write(strData)
jsonFile.close()
csvFile.close()
def jsonFileMov(self):
# 디렉토리 이동
os.chdir(self._targetPath)
# file list
fileList = os.listdir()
for f in fileList:
fileAbsPath = os.path.abspath(f)
fileName, fileExtension = os.path.splitext(fileAbsPath)
if fileExtension == ".json":
try:
shutil.move("{}.json".format(fileName), self._targetPath + "\\" + self._jsonPath)
except OSError as E:
print("파일 이동 에러")
print(E)
exit(1)
else:
print ("이동 성공")
def logstashInsert(self):
# 디렉토리 이동
os.chdir(self._targetPath + "\\" + self._jsonPath)
# file list
fileList = os.listdir()
for f in fileList:
fileAbsPath = os.path.abspath(f)
command = self._logstashRun + " -f " + self._targetConfDir + " < " + fileAbsPath
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
stdout, _ = p.communicate()
print(stdout)
print("indexing complate")
print("=======================================================")
time.sleep(5)
def main():
o = ConvJson()
o.csvRead()
o.jsonFileMov()
if __name__ == "__main__":
main()
'언어 > python' 카테고리의 다른 글
python으로 pdf 파일 read (0) | 2019.12.08 |
---|---|
백준 2108 (0) | 2019.12.08 |
네이버 기사 크롤링 => elasticsearch 적재 (0) | 2019.07.12 |
naver music 크롤링 + elasticsearch (0) | 2019.05.22 |
네이버 뉴스 크롤링 + 형태소 (0) | 2019.05.01 |