dms-client/util/file_util.py

1054 lines
46 KiB
Python
Raw Normal View History

"""
Author : XinYi Song
Time : 2022/2/12 17:13
Desc:
"""
import os
import time
import requests
from application.settings import Config
from common.tools.dms import dms_login, dms_task_record, dms_sensing_data
from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \
GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData
from scan_data.example import GetJPSSData
from upload.upload_client import upload_file_client
from util.http_file_upload import upload_client
from util.http_util import httpUtil
from util.remote_sensing_util import gf4_pmi_001
from util.xml_util import dict_to_xml
from apscheduler.schedulers.blocking import BlockingScheduler
sched = BlockingScheduler()
def list_dir(file_dir):
"""
通过 listdir 得到的是仅当前路径下的文件名不包括子目录中的文件如果需要得到所有文件需要递归
"""
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return
fileNameList = []
dms_list = dms_sensing_data(token_s)
for dms in dms_list:
fileNameList.append(dms['fileName'])
dir_list = os.listdir(file_dir)
# 判断扫描出的文件和已有的文件,将多出的文件进行解析
d = [y for y in dir_list if y not in fileNameList]
if d is None or len(d) == 0:
print('没有多余的遥感数据文件,终止程序')
return
file_total_size = ""
file_total_name = ""
xmlPath = '' # 解析出的xml文件保存的路径
ThumbnailPath = '' # 解析出的缩略图保存的路径
for cur_file in d:
# 获取文件的绝对路径
path = os.path.join(file_dir, cur_file)
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff'
xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml'
# 解析遥感数据文件demo
gf4_sensing = gf4_pmi_001(file_name, xml_path)
file_name = os.path.basename(path).split('.')[0]
xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml'
# 将遥感数据写入xml中并保存本地文件夹中
dict_to_xml(gf4_sensing, xml_path)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(xml_path, 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = gf4_sensing['CollectionCode']
DirectoryDepth = gf4_sensing['DirectoryDepth']
StartTime = gf4_sensing['StartTime']
uc = upload_client(path, DirectoryDepth, StartTime)
StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
print("{0} : is file!".format(cur_file))
if os.path.isdir(path):
print("{0} : is dir!".format(cur_file))
list_dir(path) # 递归子目录
if uc['type'] == 'ok':
continue
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
def scan_VJ102_dir(collectionCode, path):
"""
解析JPSS-VJ102元数据
:return:
"""
print('开始扫描VJ102IMG数据集')
#collectionCode = 'VJ102IMG'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = JPSSData_dict['CollectionCode']
DirectoryDepth = JPSSData_dict['DirectoryDepth']
StartTime = JPSSData_dict['StartTime']
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_VJ103_dir(collectionCode, path):
"""
解析JPSS-VJ103元数据
:param file_dir:
:return:
"""
# file_dir = 'E:/数管'
file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS'
print('开始扫描VJ103IMG数据集')
#collectionCode = 'VJ103IMG'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = JPSSData_dict['CollectionCode']
DirectoryDepth = JPSSData_dict['DirectoryDepth']
StartTime = JPSSData_dict['StartTime']
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_GF1_PMS2_dir(collectionCode, path):
"""
获取高分 PMS卫星元数据
:return:
"""
print('开始扫描GF1_PMS2_001数据集')
#collectionCode = 'GF1_PMS2_001'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
print(GFPMS_dict['xmlPath'])
files = {'file': open(GFPMS_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GFPMS_dict['CollectionCode']
DirectoryDepth = GFPMS_dict['DirectoryDepth']
StartTime = GFPMS_dict['StartTime']
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_GF3MDJ_dir(collectionCode, path):
"""
获取高分3号MDJGF-3 MDJ卫星元数据
:return:
"""
# file_dir = 'E:/数管'
print('开始扫描GF3_MDJ_SS数据集')
#collectionCode = 'GF3_MDJ_SS'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GF3_MDJ_SS_dict['CollectionCode']
DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth']
StartTime = GF3_MDJ_SS_dict['StartTime']
# uc = upload_client(path, DirectoryDepth, StartTime[0:19])
uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode)
StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_H08_dir(collectionCode, path):
"""
获取葵花8卫星元数据
:return:
"""
print('开始扫描H08数据集')
#collectionCode = 'NC_H08'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetH08_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetH08_dict['CollectionCode']
DirectoryDepth = GetH08_dict['DirectoryDepth']
StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19]
# uc = upload_client(path, DirectoryDepth, StartTime)
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(
time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(
time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_Sentinel1_dir(collectionCode, path):
"""
获取哨兵1号卫星元数据
:return:
"""
# file_dir = 'E:/数管'
print('开始扫描Sentinel1数据集')
#collectionCode = 'S1A_IW_GRDH'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = Sentinel1_dict['CollectionCode']
DirectoryDepth = Sentinel1_dict['DirectoryDepth']
StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19]
# uc = upload_client(path, DirectoryDepth, StartTime)
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(
time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_Sentinel2_dir(collectionCode, path):
"""
获取哨兵2号卫星元数据
:return:
"""
# file_dir = 'E:/数管'
print('开始扫描Sentinel2数据集')
#collectionCode = 'S2B'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = Sentinel2_dict['CollectionCode']
DirectoryDepth = Sentinel2_dict['DirectoryDepth']
StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19]
# uc = upload_client(path, DirectoryDepth, StartTime)
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(
time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_Sentinel3OL_dir(collectionCode, path):
"""
获取哨兵3号卫星元数据
:return:
"""
# file_dir = 'E:/数管'
print('开始扫描Sentinel3数据集')
#collectionCode = 'Sentinel3_OLCI'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath)
print(Sentinel3OL_dict)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = Sentinel3OL_dict['CollectionCode']
DirectoryDepth = Sentinel3OL_dict['DirectoryDepth']
StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19]
# uc = upload_client(path, DirectoryDepth, StartTime)
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(
time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_HJ1_dir(collectionCode, path):
"""
获取环境1号卫星元数据
:return:
"""
# file_dir = 'E:/数管'
#file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1'
print('开始扫描环境1号数据集')
#collectionCode = 'HJ-1'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetHJ1Data_dict['CollectionCode']
DirectoryDepth = GetHJ1Data_dict['DirectoryDepth']
StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19]
# uc = upload_client(path, DirectoryDepth, StartTime)
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(
time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(
time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_ZY3_dir(collectionCode, path):
"""
获取资源3号卫星元数据
:return:
"""
# file_dir = 'E:/数管'
#file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3'
print('开始扫描资源3号数据集')
#collectionCode = 'ZY-3'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetZY3Data_dict['CollectionCode']
DirectoryDepth = GetZY3Data_dict['DirectoryDepth']
StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19]
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(
time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(
time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
def scan_SNPP_dir(collectionCode, path):
"""
获取资源3号卫星元数据
:return:
"""
# file_dir = 'E:/数管'
#file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS'
print('开始扫描VNP02IMG数据集')
#collectionCode = 'VNP02IMG'
# 用户登录
token_s = dms_login()
# 判断定时任务是否在进行
task = dms_task_record(token_s, collectionCode)
# 如果不是空说明正在进行
if task is not None and len(task) > 0:
return 1
file_total_size = ""
file_total_name = ""
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
# 解析遥感数据文件demo
GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath)
print(GetSNPPData_dict)
# 配置文件服务器参数
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
xml = requests.post(url, data=options, files=files)
url = Config.DFS_UPLOAD_URL
files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')}
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
# 上传生成的xml文件到文件服务器
ThumbnailName = requests.post(url, data=options, files=files)
CollectionCode = GetSNPPData_dict['CollectionCode']
DirectoryDepth = GetSNPPData_dict['DirectoryDepth']
StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19]
# uc = upload_client(path, DirectoryDepth, StartTime, collectionCode)
uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode)
StartTime = time.mktime(
time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19],
'%Y-%m-%d %H:%M:%S'))
# 入库遥感数据
res_data = httpUtil(url=Config.RESING_DATA_URL,
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
"shootingTimeEndTs": EndTime,
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
"fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'],
"metaInformationFile": xml.json()['path'],
"thumbnailFile": ThumbnailName.json()['path'],
"remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']},
token=token_s).post_no_patam_herder()
print(res_data.json()['data'])
file_total_size = file_total_size + str(uc['file_size'])
file_total_size = file_total_size + ","
file_total_name = file_total_name + uc['fileName']
file_total_name = file_total_name + ","
# print("========"+suffix)
# 添加遥感数据归档任务
res = httpUtil(url=Config.DATA_TASK_URL,
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
task_code = res.json()['data']
# 结束遥感数据归档任务
header = {"Authorization": token_s}
res = requests.post(url=Config.DATA_END_TASK_URL,
params={"taskCode": task_code}, headers=header).json()
if res_data.json()['status'] == 0:
return 0
else:
return 2
if __name__ == '__main__':
# file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata'
# list_dir(file_dir)
# file_dir = 'E:/数管'
#
scan_VJ102_dir()
#
# scan_VJ103_dir()
# scan_GF1_PMS2_dir()
# scan_GF3MDJ_dir()
# scan_H08_dir()
# scan_Sentinel1_dir()
# scan_Sentinel2_dir()
# scan_Sentinel3OL_dir()
# scan_HJ1_dir()
# scan_ZY3_dir()
# scan_SNPP_dir()