#!/usr/bin/python # -*- coding: utf-8 -*- # @version : 1.0 # @Create Time : 2021/10/18 22:18 # @File : crud.py # @IDE : PyCharm # @desc : 数据库 增删改查操作 import json import os from enum import Enum from typing import Any from redis.asyncio import Redis from fastapi.encoders import jsonable_encoder from motor.motor_asyncio import AsyncIOMotorDatabase from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload from application.settings import STATIC_ROOT, SUBSCRIBE, REDIS_DB_ENABLE from core.database import redis_getter from core.mongo_manage import MongoManage from utils.file.file_manage import FileManage from . import models, schemas from core.crud import DalBase from core.exception import CustomException from utils import status from fastapi import Request class DictTypeDal(DalBase): def __init__(self, db: AsyncSession): super(DictTypeDal, self).__init__() self.db = db self.model = models.VadminDictType self.schema = schemas.DictTypeSimpleOut async def get_dicts_details(self, dict_types: list[str]) -> dict: """ 获取多个字典类型下的字典元素列表 """ data = {} options = [joinedload(self.model.details)] objs = await DictTypeDal(self.db).get_datas( limit=0, v_return_objs=True, v_options=options, dict_type=("in", dict_types) ) for obj in objs: if not obj: data[obj.dict_type] = [] continue else: data[obj.dict_type] = [schemas.DictDetailsSimpleOut.model_validate(i).model_dump() for i in obj.details] return data async def get_select_datas(self) -> list: """获取选择数据,全部数据""" sql = select(self.model) queryset = await self.db.execute(sql) return [schemas.DictTypeOptionsOut.model_validate(i).model_dump() for i in queryset.scalars().all()] class DictDetailsDal(DalBase): def __init__(self, db: AsyncSession): super(DictDetailsDal, self).__init__() self.db = db self.model = models.VadminDictDetails self.schema = schemas.DictDetailsSimpleOut class SettingsDal(DalBase): def __init__(self, db: AsyncSession): super(SettingsDal, self).__init__() self.db = db self.model = models.VadminSystemSettings self.schema = schemas.SettingsSimpleOut async def get_tab_values(self, tab_id: int) -> dict: """ 获取系统配置标签下的信息 """ datas = await self.get_datas(limit=0, tab_id=tab_id, v_return_objs=True) result = {} for data in datas: if not data.disabled: result[data.config_key] = data.config_value return result async def update_datas(self, datas: dict, request: Request) -> None: """ 更新系统配置信息 更新ico图标步骤:先将文件上传到本地,然后点击提交后,获取到文件地址,将上传的新文件覆盖原有文件 原因:ico图标的路径是在前端的index.html中固定的,所以目前只能改变图片,不改变路径 """ for key, value in datas.items(): if key == "web_ico": continue elif key == "web_ico_local_path": if not value: continue ico = await self.get_data(config_key="web_ico", tab_id=1) web_ico = datas.get("web_ico") if ico.config_value == web_ico: continue # 将上传的ico路径替换到static/system/favicon.ico文件 await FileManage.async_copy_file(value, os.path.join(STATIC_ROOT, "system/favicon.ico")) sql = update(self.model).where(self.model.config_key == "web_ico").values(config_value=web_ico) await self.db.execute(sql) else: sql = update(self.model).where(self.model.config_key == str(key)).values(config_value=value) await self.db.execute(sql) if "wx_server_app_id" in datas and REDIS_DB_ENABLE: rd = redis_getter(request) await rd.client().set("wx_server", json.dumps(datas)) elif "sms_access_key" in datas and REDIS_DB_ENABLE: rd = redis_getter(request) await rd.client().set('aliyun_sms', json.dumps(datas)) async def get_base_config(self) -> dict: """ 获取系统基本信息 """ ignore_configs = ["wx_server_app_id", "wx_server_app_secret"] datas = await self.get_datas(limit=0, tab_id=("in", ["1", "9"]), disabled=False, v_return_objs=True) result = {} for config in datas: if config.config_key not in ignore_configs: result[config.config_key] = config.config_value return result class SettingsTabDal(DalBase): def __init__(self, db: AsyncSession): super(SettingsTabDal, self).__init__(db, models.VadminSystemSettingsTab, schemas.SettingsTabSimpleOut) async def get_classify_tab_values(self, classify: list[str], hidden: bool | None = False) -> dict: """ 获取系统配置分类下的标签信息 """ model = models.VadminSystemSettingsTab options = [joinedload(model.settings)] datas = await self.get_datas( limit=0, v_options=options, classify=("in", classify), disabled=False, v_return_objs=True, hidden=hidden ) return self.__generate_values(datas) async def get_tab_name_values(self, tab_names: list[str], hidden: bool | None = False) -> dict: """ 获取系统配置标签下的标签信息 """ model = models.VadminSystemSettingsTab options = [joinedload(model.settings)] datas = await self.get_datas( limit=0, v_options=options, tab_name=("in", tab_names), disabled=False, v_return_objs=True, hidden=hidden ) return self.__generate_values(datas) @classmethod def __generate_values(cls, datas: list[models.VadminSystemSettingsTab]) -> dict: """ 生成字典值 """ return { tab.tab_name: { item.config_key: item.config_value for item in tab.settings if not item.disabled } for tab in datas } class TaskDal(MongoManage): class JobOperation(Enum): add = "add_job" def __init__(self, db: AsyncIOMotorDatabase): super(TaskDal, self).__init__(db, "vadmin_system_task", schemas.TaskSimpleOut) async def get_task( self, _id: str = None, v_return_none: bool = False, v_schema: Any = None, **kwargs ) -> dict | None: """ 获取单个数据,默认使用 ID 查询,否则使用关键词查询 包括临时字段 last_run_datetime,is_active is_active: 只有在 scheduler_task_jobs 任务运行表中存在相同 _id 才表示任务添加成功,任务状态才为 True last_run_datetime: 在 scheduler_task_record 中获取该任务最近一次执行完成的时间 :param _id: 数据 ID :param v_return_none: 是否返回空 None,否则抛出异常,默认抛出异常 :param v_schema: 指定使用的序列化对象 """ if _id: kwargs["_id"] = ("ObjectId", _id) params = self.filter_condition(**kwargs) pipeline = [ { '$addFields': { 'str_id': {'$toString': '$_id'} } }, { '$lookup': { 'from': 'scheduler_task_jobs', 'localField': 'str_id', 'foreignField': '_id', 'as': 'matched_jobs' } }, { '$lookup': { 'from': 'scheduler_task_record', 'localField': 'str_id', 'foreignField': 'job_id', 'as': 'matched_records' } }, { '$addFields': { 'is_active': { '$cond': { 'if': {'$ne': ['$matched_jobs', []]}, 'then': True, 'else': False } }, 'last_run_datetime': { '$ifNull': [ {'$arrayElemAt': ['$matched_records.create_datetime', -1]}, None ] } } }, { '$business': { 'matched_records': 0, 'matched_jobs': 0 } }, { '$match': params }, { '$facet': { 'documents': [ {'$limit': 1}, ] } } ] # 执行聚合查询 cursor = self.collection.aggregate(pipeline) result = await cursor.to_list(length=None) data = result[0]['documents'] if not data and v_return_none: return None elif not data: raise CustomException("未查找到对应数据", code=status.HTTP_404_NOT_FOUND) data = data[0] if data and v_schema: return jsonable_encoder(v_schema(**data)) return data async def get_tasks( self, page: int = 1, limit: int = 10, v_schema: Any = None, v_order: str = None, v_order_field: str = None, **kwargs ) -> tuple: """ 获取任务信息列表 添加了两个临时字段 is_active: 只有在 scheduler_task_jobs 任务运行表中存在相同 _id 才表示任务添加成功,任务状态才为 True last_run_datetime: 在 scheduler_task_record 中获取该任务最近一次执行完成的时间 """ v_order_field = v_order_field if v_order_field else 'create_datetime' v_order = -1 if v_order in self.ORDER_FIELD else 1 params = self.filter_condition(**kwargs) pipeline = [ { '$addFields': { 'str_id': {'$toString': '$_id'} } }, { '$lookup': { 'from': 'scheduler_task_jobs', 'localField': 'str_id', 'foreignField': '_id', 'as': 'matched_jobs' } }, { '$lookup': { 'from': 'scheduler_task_record', 'localField': 'str_id', 'foreignField': 'job_id', 'as': 'matched_records' } }, { '$addFields': { 'is_active': { '$cond': { 'if': {'$ne': ['$matched_jobs', []]}, 'then': True, 'else': False } }, 'last_run_datetime': { '$ifNull': [ {'$arrayElemAt': ['$matched_records.create_datetime', -1]}, None ] } } }, { '$business': { 'matched_records': 0, 'matched_jobs': 0 } }, { '$match': params }, { '$facet': { 'documents': [ {'$sort': {v_order_field: v_order}}, {'$limit': limit}, {'$skip': (page - 1) * limit} ], 'count': [{'$count': 'total'}] } } ] # 执行聚合查询 cursor = self.collection.aggregate(pipeline) result = await cursor.to_list(length=None) datas = result[0]['documents'] count = result[0]['count'][0]['total'] if result[0]['count'] else 0 if count == 0: return [], 0 elif v_schema: datas = [jsonable_encoder(v_schema(**data)) for data in datas] elif self.schema: datas = [jsonable_encoder(self.schema(**data)) for data in datas] return datas, count async def add_task(self, rd: Redis, data: dict) -> int: """ 添加任务到消息队列 使用消息无保留策略:无保留是指当发送者向某个频道发送消息时,如果没有订阅该频道的调用方,就直接将该消息丢弃。 :param rd: redis 对象 :param data: 行数据字典 :return: 接收到消息的订阅者数量。 """ exec_strategy = data.get("exec_strategy") job_params = { "name": data.get("_id"), "job_class": data.get("job_class"), "expression": data.get("expression") } if exec_strategy == "interval" or exec_strategy == "cron": job_params["start_date"] = data.get("start_date") job_params["end_date"] = data.get("end_date") message = { "operation": self.JobOperation.add.value, "task": { "exec_strategy": data.get("exec_strategy"), "job_params": job_params } } return await rd.publish(SUBSCRIBE, json.dumps(message).encode('utf-8')) async def create_task(self, rd: Redis, data: schemas.Task) -> dict: """ 创建任务 """ data_dict = data.model_dump() is_active = data_dict.pop('is_active') insert_result = await super().create_data(data_dict) obj = await self.get_task(insert_result.inserted_id, v_schema=schemas.TaskSimpleOut) # 如果分组不存在则新增分组 group = await TaskGroupDal(self.db).get_data(value=data.group, v_return_none=True) if not group: await TaskGroupDal(self.db).create_data({"value": data.group}) result = { "subscribe_number": 0, "is_active": is_active } if is_active: # 创建任务成功后, 如果任务状态为 True,则向消息队列中发送任务 result['subscribe_number'] = await self.add_task(rd, obj) return result async def put_task(self, rd: Redis, _id: str, data: schemas.Task) -> dict: """ 更新任务 """ data_dict = data.model_dump() is_active = data_dict.pop('is_active') await super(TaskDal, self).put_data(_id, data) obj: dict = await self.get_task(_id, v_schema=schemas.TaskSimpleOut) # 如果分组不存在则新增分组 group = await TaskGroupDal(self.db).get_data(value=data.group, v_return_none=True) if not group: await TaskGroupDal(self.db).create_data({"value": data.group}) try: # 删除正在运行中的 Job await SchedulerTaskJobsDal(self.db).delete_data(_id) except CustomException as e: pass result = { "subscribe_number": 0, "is_active": is_active } if is_active: # 更新任务成功后, 如果任务状态为 True,则向消息队列中发送任务 result['subscribe_number'] = await self.add_task(rd, obj) return result async def delete_task(self, _id: str) -> bool: """ 删除任务 """ result = await super(TaskDal, self).delete_data(_id) try: # 删除正在运行中的 Job await SchedulerTaskJobsDal(self.db).delete_data(_id) except CustomException as e: pass return result async def run_once_task(self, rd: Redis, _id: str) -> int: """ 执行一次任务 """ obj: dict = await self.get_data(_id, v_schema=schemas.TaskSimpleOut) message = { "operation": self.JobOperation.add.value, "task": { "exec_strategy": "once", "job_params": { "name": obj.get("_id"), "job_class": obj.get("job_class") } } } return await rd.publish(SUBSCRIBE, json.dumps(message).encode('utf-8')) class TaskGroupDal(MongoManage): def __init__(self, db: AsyncIOMotorDatabase): super(TaskGroupDal, self).__init__(db, "vadmin_system_task_group") class TaskRecordDal(MongoManage): def __init__(self, db: AsyncIOMotorDatabase): super(TaskRecordDal, self).__init__(db, "scheduler_task_record") class SchedulerTaskJobsDal(MongoManage): def __init__(self, db: AsyncIOMotorDatabase): super(SchedulerTaskJobsDal, self).__init__(db, "scheduler_task_jobs", is_object_id=False)