1195 lines
58 KiB
Python
1195 lines
58 KiB
Python
|
"""
|
|||
|
|
Author : XinYi Song
|
|||
|
|
Time : 2021/11/3 9:00
|
|||
|
|
Desc:
|
|||
|
|
"""
|
|||
|
|
import os
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
import requests
|
|||
|
|
|
|||
|
|
from application.settings import Config
|
|||
|
|
from common.tools.dms import dms_login, dms_task_record, dms_sensing_data
|
|||
|
|
from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \
|
|||
|
|
GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData
|
|||
|
|
from scan_data.example import GetJPSSData
|
|||
|
|
from util.http_file_upload import upload_client
|
|||
|
|
from util.http_util import httpUtil
|
|||
|
|
from util.remote_sensing_util import gf4_pmi_001
|
|||
|
|
from util.xml_util import dict_to_xml
|
|||
|
|
from apscheduler.schedulers.blocking import BlockingScheduler
|
|||
|
|
|
|||
|
|
sched = BlockingScheduler()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def list_dir(file_dir):
|
|||
|
|
"""
|
|||
|
|
通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归
|
|||
|
|
"""
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = '' # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = '' # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff'
|
|||
|
|
xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml'
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
gf4_sensing = gf4_pmi_001(file_name, xml_path)
|
|||
|
|
file_name = os.path.basename(path).split('.')[0]
|
|||
|
|
xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml'
|
|||
|
|
# 将遥感数据写入xml中，并保存本地文件夹中
|
|||
|
|
dict_to_xml(gf4_sensing, xml_path)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(xml_path, 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = gf4_sensing['CollectionCode']
|
|||
|
|
DirectoryDepth = gf4_sensing['DirectoryDepth']
|
|||
|
|
StartTime = gf4_sensing['StartTime']
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_VJ102_dir():
|
|||
|
|
"""
|
|||
|
|
解析JPSS-VJ102元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描VJ102IMG数据集')
|
|||
|
|
collectionCode = 'VJ102IMG'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'VJ102IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
print(path)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = JPSSData_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = JPSSData_dict['DirectoryDepth']
|
|||
|
|
StartTime = JPSSData_dict['StartTime']
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_VJ103_dir():
|
|||
|
|
"""
|
|||
|
|
解析JPSS-VJ103元数据
|
|||
|
|
:param file_dir:
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描VJ103IMG数据集')
|
|||
|
|
collectionCode = 'VJ103IMG'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'VJ103IMG' in cur_file and os.path.splitext(cur_file)[1] == '.nc':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(JPSSData_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = JPSSData_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = JPSSData_dict['DirectoryDepth']
|
|||
|
|
StartTime = JPSSData_dict['StartTime']
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_GF1_PMS2_dir():
|
|||
|
|
"""
|
|||
|
|
获取高分 PMS卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描GF1_PMS2_001数据集')
|
|||
|
|
collectionCode = 'GF1_PMS2_001'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'GF1_PMS2' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.gz':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
print(GFPMS_dict['xmlPath'])
|
|||
|
|
files = {'file': open(GFPMS_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = GFPMS_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = GFPMS_dict['DirectoryDepth']
|
|||
|
|
StartTime = GFPMS_dict['StartTime']
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_GF3MDJ_dir():
|
|||
|
|
"""
|
|||
|
|
获取高分3号MDJ（GF-3 MDJ）卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描GF3_MDJ_SS数据集')
|
|||
|
|
collectionCode = 'GF3_MDJ_SS'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'GF3_MDJ' in cur_file[0:7] and os.path.splitext(cur_file)[1] == '.gz':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = GF3_MDJ_SS_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth']
|
|||
|
|
StartTime = GF3_MDJ_SS_dict['StartTime']
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime[0:19])
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_H08_dir():
|
|||
|
|
"""
|
|||
|
|
获取高分3号MDJ（GF-3 MDJ）卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描H08数据集')
|
|||
|
|
collectionCode = 'NC_H08'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'NC_H08' in cur_file[0:6] and os.path.splitext(cur_file)[1] == '.nc':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetH08_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = GetH08_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = GetH08_dict['DirectoryDepth']
|
|||
|
|
StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(GetH08_dict['ProduceTime'][0:10] + ' '+ GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_Sentinel1_dir():
|
|||
|
|
"""
|
|||
|
|
获取哨兵1号卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描Sentinel1数据集')
|
|||
|
|
collectionCode = 'S1A_IW_GRDH'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'S1A_IW_GRDH' in cur_file[0:11] and os.path.splitext(cur_file)[1] == '.zip':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = Sentinel1_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = Sentinel1_dict['DirectoryDepth']
|
|||
|
|
StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_Sentinel2_dir():
|
|||
|
|
"""
|
|||
|
|
获取哨兵2号卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描Sentinel2数据集')
|
|||
|
|
collectionCode = 'S2B'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'S2B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = Sentinel2_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = Sentinel2_dict['DirectoryDepth']
|
|||
|
|
StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_Sentinel3OL_dir():
|
|||
|
|
"""
|
|||
|
|
获取哨兵3号卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描Sentinel3数据集')
|
|||
|
|
collectionCode = 'Sentinel3_OLCI'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'S3B' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.zip':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
print(path)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath)
|
|||
|
|
print(Sentinel3OL_dict)
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = Sentinel3OL_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = Sentinel3OL_dict['DirectoryDepth']
|
|||
|
|
StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_HJ1_dir():
|
|||
|
|
"""
|
|||
|
|
获取环境1号卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描环境1号数据集')
|
|||
|
|
collectionCode = 'HJ-1'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'HJ1' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath)
|
|||
|
|
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = GetHJ1Data_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = GetHJ1Data_dict['DirectoryDepth']
|
|||
|
|
StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_ZY3_dir():
|
|||
|
|
"""
|
|||
|
|
获取资源3号卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描资源3号数据集')
|
|||
|
|
collectionCode = 'ZY-3'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'ZY3' in cur_file[0:3] and os.path.splitext(cur_file)[1] == '.gz':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath)
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = GetZY3Data_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = GetZY3Data_dict['DirectoryDepth']
|
|||
|
|
StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scan_SNPP_dir():
|
|||
|
|
"""
|
|||
|
|
获取资源3号卫星元数据
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
file_dir = 'E:/数管'
|
|||
|
|
print('开始扫描VNP02IMG数据集')
|
|||
|
|
collectionCode = 'VNP02IMG'
|
|||
|
|
# 用户登录
|
|||
|
|
token_s = dms_login()
|
|||
|
|
# 判断定时任务是否在进行
|
|||
|
|
task = dms_task_record(token_s, collectionCode)
|
|||
|
|
# 如果不是空说明正在进行
|
|||
|
|
if task is not None and len(task) > 0:
|
|||
|
|
return
|
|||
|
|
fileNameList = []
|
|||
|
|
dms_list = dms_sensing_data(token_s, collectionCode)
|
|||
|
|
for dms in dms_list:
|
|||
|
|
fileNameList.append(dms['fileName'])
|
|||
|
|
dir_list = os.listdir(file_dir)
|
|||
|
|
# 判断扫描出的文件和已有的文件，将多出的文件进行解析
|
|||
|
|
d = [y for y in dir_list if y not in fileNameList]
|
|||
|
|
if d is None or len(d) == 0:
|
|||
|
|
print('没有多余的遥感数据文件，终止程序')
|
|||
|
|
return
|
|||
|
|
file_total_size = ""
|
|||
|
|
file_total_name = ""
|
|||
|
|
xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径
|
|||
|
|
ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径
|
|||
|
|
for cur_file in d:
|
|||
|
|
if 'VNP02IMG' in cur_file[0:8] and os.path.splitext(cur_file)[1] == '.nc':
|
|||
|
|
|
|||
|
|
# 获取文件的绝对路径
|
|||
|
|
path = os.path.join(file_dir, cur_file)
|
|||
|
|
if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径
|
|||
|
|
# 解析遥感数据文件（demo）
|
|||
|
|
GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath)
|
|||
|
|
# 配置文件服务器参数
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
xml = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
url = Config.DFS_UPLOAD_URL
|
|||
|
|
files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')}
|
|||
|
|
options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode}
|
|||
|
|
# 上传生成的xml文件到文件服务器
|
|||
|
|
ThumbnailName = requests.post(url, data=options, files=files)
|
|||
|
|
|
|||
|
|
CollectionCode = GetSNPPData_dict['CollectionCode']
|
|||
|
|
DirectoryDepth = GetSNPPData_dict['DirectoryDepth']
|
|||
|
|
StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19]
|
|||
|
|
uc = upload_client(path, DirectoryDepth, StartTime)
|
|||
|
|
|
|||
|
|
StartTime = time.mktime(time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S'))
|
|||
|
|
|
|||
|
|
# 入库遥感数据
|
|||
|
|
res_data = httpUtil(url=Config.RESING_DATA_URL,
|
|||
|
|
data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime,
|
|||
|
|
"shootingTimeEndTs": EndTime,
|
|||
|
|
"fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'],
|
|||
|
|
"fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'],
|
|||
|
|
"metaInformationFile": xml.json()['path'],
|
|||
|
|
"thumbnailFile": ThumbnailName.json()['path'],
|
|||
|
|
"remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']},
|
|||
|
|
token=token_s).post_no_patam_herder()
|
|||
|
|
print(res_data.json()['data'])
|
|||
|
|
|
|||
|
|
file_total_size = file_total_size + str(uc['file_size'])
|
|||
|
|
file_total_size = file_total_size + ","
|
|||
|
|
|
|||
|
|
file_total_name = file_total_name + uc['fileName']
|
|||
|
|
file_total_name = file_total_name + ","
|
|||
|
|
# print("========"+suffix)
|
|||
|
|
print("{0} : is file!".format(cur_file))
|
|||
|
|
if os.path.isdir(path):
|
|||
|
|
print("{0} : is dir!".format(cur_file))
|
|||
|
|
list_dir(path) # 递归子目录
|
|||
|
|
if uc['type'] == 'ok':
|
|||
|
|
continue
|
|||
|
|
# 添加遥感数据归档任务
|
|||
|
|
res = httpUtil(url=Config.DATA_TASK_URL,
|
|||
|
|
data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name,
|
|||
|
|
"storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder()
|
|||
|
|
task_code = res.json()['data']
|
|||
|
|
|
|||
|
|
# 结束遥感数据归档任务
|
|||
|
|
header = {"Authorization": token_s}
|
|||
|
|
res = requests.post(url=Config.DATA_END_TASK_URL,
|
|||
|
|
|
|||
|
|
params={"taskCode": task_code}, headers=header).json()
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
# file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata'
|
|||
|
|
# list_dir(file_dir)
|
|||
|
|
# file_dir = 'E:/数管'
|
|||
|
|
#
|
|||
|
|
# scan_VJ102_dir()
|
|||
|
|
#
|
|||
|
|
# scan_VJ103_dir()
|
|||
|
|
|
|||
|
|
# scan_GF1_PMS2_dir()
|
|||
|
|
# scan_GF3MDJ_dir()
|
|||
|
|
# scan_H08_dir()
|
|||
|
|
# scan_Sentinel1_dir()
|
|||
|
|
# scan_Sentinel2_dir()
|
|||
|
|
# scan_Sentinel3OL_dir()
|
|||
|
|
# scan_HJ1_dir()
|
|||
|
|
# scan_ZY3_dir()
|
|||
|
|
scan_SNPP_dir()
|