""" Author : XinYi Song Time : 2022/2/12 17:13 Desc: """ import os import time import requests from application.settings import Config from common.tools.dms import dms_login, dms_task_record, dms_sensing_data from scan_data.GetMetaInfo import GetGFPMSData, GetGF3MDJData, GetH08Data, GetSentinel1Data, GetSentinel2Data, \ GetSentinel3OLData, GetHJ1Data, GetZY3Data, GetSNPPData from scan_data.example import GetJPSSData from upload.upload_client import upload_file_client from util.http_file_upload import upload_client from util.http_util import httpUtil from util.remote_sensing_util import gf4_pmi_001 from util.xml_util import dict_to_xml from apscheduler.schedulers.blocking import BlockingScheduler sched = BlockingScheduler() def list_dir(file_dir): """ 通过 listdir 得到的是仅当前路径下的文件名，不包括子目录中的文件，如果需要得到所有文件需要递归 """ # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return fileNameList = [] dms_list = dms_sensing_data(token_s) for dms in dms_list: fileNameList.append(dms['fileName']) dir_list = os.listdir(file_dir) # 判断扫描出的文件和已有的文件，将多出的文件进行解析 d = [y for y in dir_list if y not in fileNameList] if d is None or len(d) == 0: print('没有多余的遥感数据文件，终止程序') return file_total_size = "" file_total_name = "" xmlPath = '' # 解析出的xml文件保存的路径 ThumbnailPath = '' # 解析出的缩略图保存的路径 for cur_file in d: # 获取文件的绝对路径 path = os.path.join(file_dir, cur_file) if os.path.isfile(path): # 判断是否是文件还是目录需要用绝对路径 file_name = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.tiff' xml_path = 'GF4_PMS_E119.8_N35.3_20210908_L1A0000417337.xml' # 解析遥感数据文件（demo） gf4_sensing = gf4_pmi_001(file_name, xml_path) file_name = os.path.basename(path).split('.')[0] xml_path = 'D:/file/work/pythonyuanma/dms_management/xml/' + file_name + '.xml' # 将遥感数据写入xml中，并保存本地文件夹中 dict_to_xml(gf4_sensing, xml_path) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(xml_path, 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(gf4_sensing['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/GF4_PMI_001'} # 参阅浏览器上传的选项 # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = gf4_sensing['CollectionCode'] DirectoryDepth = gf4_sensing['DirectoryDepth'] StartTime = gf4_sensing['StartTime'] uc = upload_client(path, DirectoryDepth, StartTime) StartTime = time.mktime(time.strptime(gf4_sensing['StartTime'], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(gf4_sensing['EndTime'], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": "GF4_PMI_001", "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": gf4_sensing['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": gf4_sensing['boundaryGeomStr']}, token=token_s).post_no_patam_herder() file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) print("{0} : is file!".format(cur_file)) if os.path.isdir(path): print("{0} : is dir!".format(cur_file)) list_dir(path) # 递归子目录 if uc['type'] == 'ok': continue # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": "GF4_PMI_001", "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_VJ102_dir(collectionCode, path): """ 解析JPSS-VJ102元数据 :return: """ print('开始扫描VJ102IMG数据集') #collectionCode = 'VJ102IMG' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = JPSSData_dict['CollectionCode'] DirectoryDepth = JPSSData_dict['DirectoryDepth'] StartTime = JPSSData_dict['StartTime'] # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_VJ103_dir(collectionCode, path): """ 解析JPSS-VJ103元数据 :param file_dir: :return: """ # file_dir = 'E:/数管' file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\JPSS' print('开始扫描VJ103IMG数据集') #collectionCode = 'VJ103IMG' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） JPSSData_dict = GetJPSSData(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(JPSSData_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(JPSSData_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = JPSSData_dict['CollectionCode'] DirectoryDepth = JPSSData_dict['DirectoryDepth'] StartTime = JPSSData_dict['StartTime'] # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) StartTime = time.mktime(time.strptime(JPSSData_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(JPSSData_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": JPSSData_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": JPSSData_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_GF1_PMS2_dir(collectionCode, path): """ 获取高分 PMS卫星元数据 :return: """ print('开始扫描GF1_PMS2_001数据集') #collectionCode = 'GF1_PMS2_001' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） GFPMS_dict = GetGFPMSData(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL print(GFPMS_dict['xmlPath']) files = {'file': open(GFPMS_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(GFPMS_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = GFPMS_dict['CollectionCode'] DirectoryDepth = GFPMS_dict['DirectoryDepth'] StartTime = GFPMS_dict['StartTime'] # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) StartTime = time.mktime(time.strptime(GFPMS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(GFPMS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": GFPMS_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": GFPMS_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_GF3MDJ_dir(collectionCode, path): """ 获取高分3号MDJ（GF-3 MDJ）卫星元数据 :return: """ # file_dir = 'E:/数管' print('开始扫描GF3_MDJ_SS数据集') #collectionCode = 'GF3_MDJ_SS' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） GF3_MDJ_SS_dict = GetGF3MDJData(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(GF3_MDJ_SS_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(GF3_MDJ_SS_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = GF3_MDJ_SS_dict['CollectionCode'] DirectoryDepth = GF3_MDJ_SS_dict['DirectoryDepth'] StartTime = GF3_MDJ_SS_dict['StartTime'] # uc = upload_client(path, DirectoryDepth, StartTime[0:19]) uc = upload_file_client(path, DirectoryDepth, StartTime[0:19], collectionCode) StartTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['StartTime'][0:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(GF3_MDJ_SS_dict['EndTime'][0:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": GF3_MDJ_SS_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": GF3_MDJ_SS_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_H08_dir(collectionCode, path): """ 获取葵花8卫星元数据 :return: """ print('开始扫描H08数据集') #collectionCode = 'NC_H08' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） GetH08_dict = GetH08Data(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(GetH08_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(GetH08_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = GetH08_dict['CollectionCode'] DirectoryDepth = GetH08_dict['DirectoryDepth'] StartTime = GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19] # uc = upload_client(path, DirectoryDepth, StartTime) uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime( time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime( time.strptime(GetH08_dict['ProduceTime'][0:10] + ' ' + GetH08_dict['ProduceTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": GetH08_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": GetH08_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_Sentinel1_dir(collectionCode, path): """ 获取哨兵1号卫星元数据 :return: """ # file_dir = 'E:/数管' print('开始扫描Sentinel1数据集') #collectionCode = 'S1A_IW_GRDH' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） Sentinel1_dict = GetSentinel1Data(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(Sentinel1_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(Sentinel1_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = Sentinel1_dict['CollectionCode'] DirectoryDepth = Sentinel1_dict['DirectoryDepth'] StartTime = Sentinel1_dict['ProduceTime'][0:10] + ' ' + Sentinel1_dict['ProduceTime'][11:19] # uc = upload_client(path, DirectoryDepth, StartTime) uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime(time.strptime(Sentinel1_dict['StartTime'][0:10] + ' ' + Sentinel1_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime( time.strptime(Sentinel1_dict['StopTime'][0:10] + ' ' + Sentinel1_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": Sentinel1_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": Sentinel1_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_Sentinel2_dir(collectionCode, path): """ 获取哨兵2号卫星元数据 :return: """ # file_dir = 'E:/数管' print('开始扫描Sentinel2数据集') #collectionCode = 'S2B' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） Sentinel2_dict = GetSentinel2Data(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(Sentinel2_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(Sentinel2_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = Sentinel2_dict['CollectionCode'] DirectoryDepth = Sentinel2_dict['DirectoryDepth'] StartTime = Sentinel2_dict['ProduceTime'][0:10] + ' ' + Sentinel2_dict['ProduceTime'][11:19] # uc = upload_client(path, DirectoryDepth, StartTime) uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime(time.strptime(Sentinel2_dict['StartTime'][0:10] + ' ' + Sentinel2_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime( time.strptime(Sentinel2_dict['StopTime'][0:10] + ' ' + Sentinel2_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": Sentinel2_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": Sentinel2_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_Sentinel3OL_dir(collectionCode, path): """ 获取哨兵3号卫星元数据 :return: """ # file_dir = 'E:/数管' print('开始扫描Sentinel3数据集') #collectionCode = 'Sentinel3_OLCI' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） Sentinel3OL_dict = GetSentinel3OLData(path, xmlPath, ThumbnailPath) print(Sentinel3OL_dict) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(Sentinel3OL_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(Sentinel3OL_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = Sentinel3OL_dict['CollectionCode'] DirectoryDepth = Sentinel3OL_dict['DirectoryDepth'] StartTime = Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19] # uc = upload_client(path, DirectoryDepth, StartTime) uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime( time.strptime(Sentinel3OL_dict['StartTime'][0:10] + ' ' + Sentinel3OL_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(Sentinel3OL_dict['StopTime'][0:10] + ' ' + Sentinel3OL_dict['StopTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": Sentinel3OL_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": Sentinel3OL_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_HJ1_dir(collectionCode, path): """ 获取环境1号卫星元数据 :return: """ # file_dir = 'E:/数管' #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\HJ-1' print('开始扫描环境1号数据集') #collectionCode = 'HJ-1' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） GetHJ1Data_dict = GetHJ1Data(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(GetHJ1Data_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(GetHJ1Data_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = GetHJ1Data_dict['CollectionCode'] DirectoryDepth = GetHJ1Data_dict['DirectoryDepth'] StartTime = GetHJ1Data_dict['ProductTime'][0:10] + ' ' + GetHJ1Data_dict['ProductTime'][11:19] # uc = upload_client(path, DirectoryDepth, StartTime) uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime( time.strptime(GetHJ1Data_dict['StartTime'][0:10] + ' ' + GetHJ1Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime( time.strptime(GetHJ1Data_dict['EndTime'][0:10] + ' ' + GetHJ1Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": GetHJ1Data_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": GetHJ1Data_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_ZY3_dir(collectionCode, path): """ 获取资源3号卫星元数据 :return: """ # file_dir = 'E:/数管' #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\ZY-3' print('开始扫描资源3号数据集') #collectionCode = 'ZY-3' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） GetZY3Data_dict = GetZY3Data(path, xmlPath, ThumbnailPath) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(GetZY3Data_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(GetZY3Data_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = GetZY3Data_dict['CollectionCode'] DirectoryDepth = GetZY3Data_dict['DirectoryDepth'] StartTime = GetZY3Data_dict['ProduceTime'][0:10] + ' ' + GetZY3Data_dict['ProduceTime'][11:19] uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime( time.strptime(GetZY3Data_dict['StartTime'][0:10] + ' ' + GetZY3Data_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime( time.strptime(GetZY3Data_dict['EndTime'][0:10] + ' ' + GetZY3Data_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": GetZY3Data_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": GetZY3Data_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() def scan_SNPP_dir(collectionCode, path): """ 获取资源3号卫星元数据 :return: """ # file_dir = 'E:/数管' #file_dir = '\\\\192.168.2.85\\数据\\不同传感器数据\\VIIRS' print('开始扫描VNP02IMG数据集') #collectionCode = 'VNP02IMG' # 用户登录 token_s = dms_login() # 判断定时任务是否在进行 task = dms_task_record(token_s, collectionCode) # 如果不是空说明正在进行 if task is not None and len(task) > 0: return file_total_size = "" file_total_name = "" xmlPath = Config.XML_PATH # 解析出的xml文件保存的路径 ThumbnailPath = Config.THUMBNAIL_PATH # 解析出的缩略图保存的路径 # 解析遥感数据文件（demo） GetSNPPData_dict = GetSNPPData(path, xmlPath, ThumbnailPath) print(GetSNPPData_dict) # 配置文件服务器参数 url = Config.DFS_UPLOAD_URL files = {'file': open(GetSNPPData_dict['xmlPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 xml = requests.post(url, data=options, files=files) url = Config.DFS_UPLOAD_URL files = {'file': open(GetSNPPData_dict['ThumbnailPath'], 'rb')} options = {'output': 'json', 'path': '/archive_data/remote_sensing_data/' + collectionCode} # 上传生成的xml文件到文件服务器 ThumbnailName = requests.post(url, data=options, files=files) CollectionCode = GetSNPPData_dict['CollectionCode'] DirectoryDepth = GetSNPPData_dict['DirectoryDepth'] StartTime = GetSNPPData_dict['ProductionTime'][0:10] + ' ' + GetSNPPData_dict['ProductionTime'][11:19] # uc = upload_client(path, DirectoryDepth, StartTime, collectionCode) uc = upload_file_client(path, DirectoryDepth, StartTime, collectionCode) StartTime = time.mktime( time.strptime(GetSNPPData_dict['StartTime'][0:10] + ' ' + GetSNPPData_dict['StartTime'][11:19], '%Y-%m-%d %H:%M:%S')) EndTime = time.mktime(time.strptime(GetSNPPData_dict['EndTime'][0:10] + ' ' + GetSNPPData_dict['EndTime'][11:19], '%Y-%m-%d %H:%M:%S')) # 入库遥感数据 res_data = httpUtil(url=Config.RESING_DATA_URL, data={"collectionCode": collectionCode, "shootingTimeStartTs": StartTime, "shootingTimeEndTs": EndTime, "fileMd5": uc['md5'], "fileName": uc['fileName'], "filePath": uc['file_path'], "fileSize": uc['file_size'], "cloudCoverage": GetSNPPData_dict['CloudPercent'], "metaInformationFile": xml.json()['path'], "thumbnailFile": ThumbnailName.json()['path'], "remarks": "", "boundaryGeomStr": GetSNPPData_dict['boundaryGeomStr']}, token=token_s).post_no_patam_herder() print(res_data.json()['data']) file_total_size = file_total_size + str(uc['file_size']) file_total_size = file_total_size + "," file_total_name = file_total_name + uc['fileName'] file_total_name = file_total_name + "," # print("========"+suffix) # 添加遥感数据归档任务 res = httpUtil(url=Config.DATA_TASK_URL, data={"clientCode": "client1", "collectionCode": collectionCode, "storageFileList": file_total_name, "storageFileSizeList": file_total_size, "remarks": ""}, token=token_s).post_no_patam_herder() task_code = res.json()['data'] # 结束遥感数据归档任务 header = {"Authorization": token_s} res = requests.post(url=Config.DATA_END_TASK_URL, params={"taskCode": task_code}, headers=header).json() if __name__ == '__main__': # file_dir = 'C:/Users/HP/Desktop/Number tube/GF4_PMI_001/sensingdata' # list_dir(file_dir) # file_dir = 'E:/数管' # scan_VJ102_dir() # # scan_VJ103_dir() # scan_GF1_PMS2_dir() # scan_GF3MDJ_dir() # scan_H08_dir() # scan_Sentinel1_dir() # scan_Sentinel2_dir() # scan_Sentinel3OL_dir() # scan_HJ1_dir() # scan_ZY3_dir() # scan_SNPP_dir()