From 7a9e571a96535505021e8f92aed35cb1f77232ab Mon Sep 17 00:00:00 2001 From: sunyugang Date: Fri, 18 Apr 2025 17:22:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E9=A1=B9=E7=9B=AE=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E6=A8=A1=E5=9D=97=E7=9A=84=E6=8E=A5=E5=8F=A3=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- application/config/development.py | 21 +++++++----- application/config/production.py | 15 ++++++--- apps/business/detect/crud.py | 7 ++-- apps/business/project/crud.py | 22 +++++++++---- .../business/project/schemas/project_image.py | 6 ++-- apps/business/project/views.py | 33 ++++++++++--------- apps/vadmin/auth/crud.py | 6 ++-- apps/vadmin/auth/models/user.py | 1 - apps/vadmin/auth/params/user.py | 2 -- apps/vadmin/auth/schemas/user.py | 2 -- apps/vadmin/auth/utils/current.py | 3 +- apps/vadmin/auth/utils/login.py | 2 -- apps/vadmin/auth/utils/validation/login.py | 2 +- apps/vadmin/system/views.py | 11 ++----- core/crud.py | 10 ++++++ utils/huawei_obs.py | 22 ++++++------- 16 files changed, 91 insertions(+), 74 deletions(-) diff --git a/application/config/development.py b/application/config/development.py index fd4ee07..29e6493 100644 --- a/application/config/development.py +++ b/application/config/development.py @@ -29,7 +29,6 @@ MONGO_DB_ENABLE = False MONGO_DB_NAME = "kinit" MONGO_DB_URL = f"mongodb://kinit:123456@177.8.0.6:27017/?authSource={MONGO_DB_NAME}" - """ 华为云对象存储OBS设置 """ @@ -47,10 +46,16 @@ HUAWEI_OBS = { IP_PARSE_ENABLE = False IP_PARSE_TOKEN = "IP_PARSE_TOKEN" - -datasets_url = f'D:\syg\yolov5\datasets' -runs_url = f'D:\syg\yolov5\runs' -detect_url = f'D:\syg\yolov5\detect' -yolo_url = f'D:\syg\workspace\aicheckv2\yolov5' - -images_url = f'D:\syg\images' +""" +yolov5的各种文件保存地址 +images_url 原始图片保存地址 +datasets_url 文件集文件地址 +runs_url 训练结果文件地址 +detect_url 推理集合文件地址 +yolo_url yolov5执行文件所在的文件路径 +""" +images_url = r'D:\syg\images' +datasets_url = r'D:\syg\yolov5\datasets' +runs_url = r'D:\syg\yolov5\runs' +detect_url = r'D:\syg\yolov5\detect' +yolo_url = r'D:\syg\workspace\aicheckv2\yolov5' \ No newline at end of file diff --git a/application/config/production.py b/application/config/production.py index bc88c64..f51eb84 100644 --- a/application/config/production.py +++ b/application/config/production.py @@ -28,7 +28,6 @@ MONGO_DB_ENABLE = False MONGO_DB_NAME = "kinit" MONGO_DB_URL = f"mongodb://kinit:123456@177.8.0.6:27017/?authSource={MONGO_DB_NAME}" - """ 华为云对象存储OBS设置 """ @@ -39,7 +38,6 @@ HUAWEI_OBS = { "bucketName": "aicheckv2" } - """ 获取IP地址归属地 文档：https://user.ip138.com/ip/doc @@ -47,9 +45,16 @@ HUAWEI_OBS = { IP_PARSE_ENABLE = False IP_PARSE_TOKEN = "IP_PARSE_TOKEN" +""" +yolov5的各种文件保存地址 +images_url 原始图片保存地址 +datasets_url 待训练文件地址 +runs_url 训练结果文件地址 +detect_url 推理集合文件地址 +yolo_url yolo执行文件所在的文件路径 +""" +images_url = f'/home/aicheckv2/images' datasets_url = f'/home/aicheckv2/yolov5/datasets' runs_url = f'/home/aicheckv2/yolov5/runs' detect_url = f'/home/aicheckv2/yolov5/detect' -yolo_url = f'/home/aicheckv2/backend/yolov5' - -images_url = f'/home/aicheckv2/images' +yolo_url = f'/home/aicheckv2/backend/yolov5' \ No newline at end of file diff --git a/apps/business/detect/crud.py b/apps/business/detect/crud.py index 5a1de23..0bae20c 100644 --- a/apps/business/detect/crud.py +++ b/apps/business/detect/crud.py @@ -12,7 +12,7 @@ from . import schemas, models from utils.random_utils import random_str from utils import os_utils as os from application.settings import detect_url -from utils.huawei_obs import ObsClient +from utils.huawei_obs import MyObs from utils import status from core.exception import CustomException @@ -76,6 +76,7 @@ class ProjectDetectFileDal(DalBase): async def add_file(self, detect: models.ProjectDetect, files: list[UploadFile]): images = [] + obs = MyObs() for file in files: image = models.ProjectDetectFile() image.detect_id = detect.id @@ -85,7 +86,7 @@ class ProjectDetectFileDal(DalBase): image.image_url = path # 上传到obs object_key = detect.detect_no + '/' + file.filename - success, key, url = ObsClient.put_file(object_key=object_key, file_path=path) + success, key, url = obs.put_file(object_key=object_key, file_path=path) if success: image.object_key = object_key image.thumb_image_url = url @@ -103,7 +104,7 @@ class ProjectDetectFileDal(DalBase): file_urls.append(file.file_url) object_keys.append(file.object_key) os.delete_paths(file_urls) - ObsClient.del_objects(object_keys) + MyObs().del_objects(object_keys) await self.delete_datas(ids, v_soft=False) class ProjectDetectLogDal(DalBase): diff --git a/apps/business/project/crud.py b/apps/business/project/crud.py index 0c0534c..b501054 100644 --- a/apps/business/project/crud.py +++ b/apps/business/project/crud.py @@ -8,7 +8,7 @@ from . import schemas, models, params from apps.vadmin.auth.utils.validation.auth import Auth from utils import os_utils as os, random_utils as ru -from utils.huawei_obs import ObsClient +from utils.huawei_obs import MyObs from utils import status from core.exception import CustomException from application.settings import datasets_url, runs_url, images_url @@ -169,6 +169,7 @@ class ProjectImageDal(DalBase): 上传项目图片 """ image_models = [] + obs = MyObs() for file in files: image = models.ProjectImage() image.project_id = pro.id @@ -179,14 +180,14 @@ class ProjectImageDal(DalBase): image.image_url = path # 上传图片到obs object_key = pro.project_no + '/' + img_type + '/' + file.filename - success, key, url = ObsClient.put_file(object_key=object_key, file_path=path) + success, key, url = obs.put_file(object_key=object_key, file_path=path) if success: image.object_key = object_key image.thumb_image_url = url else: raise CustomException("obs上传失败", code=status.HTTP_ERROR) image_models.append(image) - await self.create_datas(datas=image_models) + await self.create_models(datas=image_models) return len(image_models) async def check_img_name(self, file_name: str, project_id: int, img_type: str): @@ -207,12 +208,12 @@ class ProjectImageDal(DalBase): file_urls = [] object_keys = [] for img_id in ids: - image = self.get_data(data_id=img_id) + image = await self.get_data(data_id=img_id) if image: file_urls.append(image.image_url) object_keys.append(image.object_key) - os.delete_file_if_exists(**file_urls) - ObsClient.del_objects(object_keys) + os.delete_file_if_exists(*file_urls) + MyObs().del_objects(object_keys) await self.delete_datas(ids) async def get_img_count( @@ -330,6 +331,11 @@ class ProjectImgLabelDal(DalBase): v_order="asc", v_order_field="id") + async def del_img_label(self, label_ids: list[int]): + img_labels = self.get_datas(v_where=[self.model.label_id.in_(label_ids)]) + img_label_ids = [i.id for i in img_labels] + self.delete_datas(ids=img_label_ids) + class ProjectImgLeaferDal(DalBase): """ @@ -342,6 +348,10 @@ class ProjectImgLeaferDal(DalBase): self.model = models.ProjectImgLeafer self.schema = schemas.ProjectImgLeaferOut + async def get_leafer(self, image_id: int): + img_label = self.get_data(v_where=[self.model.image_id == image_id]) + return img_label.leafer + async def add_leafer(self, img_label_in: schemas.ProjectImgLeaferLabel): # 先把历史数据都删掉，然后再保存 image_id = img_label_in.image_id diff --git a/apps/business/project/schemas/project_image.py b/apps/business/project/schemas/project_image.py index 184a824..2ae4779 100644 --- a/apps/business/project/schemas/project_image.py +++ b/apps/business/project/schemas/project_image.py @@ -16,7 +16,7 @@ class ProjectImage(BaseModel): project_id: Optional[int] = Field(..., description="项目id") file_name: Optional[str] = Field(None, description="文件名称") thumb_image_url: Optional[str] = Field(None, description="图片在obs上的链接") - create_time: DatetimeStr + create_datetime: DatetimeStr model_config = ConfigDict(from_attributes=True) @@ -26,7 +26,7 @@ class ProjectImageOut(BaseModel): project_id: Optional[int] = Field(..., description="项目id") file_name: Optional[str] = Field(None, description="文件名称") thumb_image_url: Optional[str] = Field(None, description="图片在obs上的链接") - create_time: DatetimeStr - label_count: Optional[int] + label_count: Optional[int] = Field(0, description="图片的标注数量") + create_datetime: DatetimeStr model_config = ConfigDict(from_attributes=True) diff --git a/apps/business/project/views.py b/apps/business/project/views.py index 4662dca..9cef94b 100644 --- a/apps/business/project/views.py +++ b/apps/business/project/views.py @@ -5,11 +5,11 @@ # @File : views.py # @IDE : PyCharm # @desc : 路由，项目信息管理，包括项目主体，项目图片，项目标签和图片的标注信息 -from utils.response import SuccessResponse, ErrorResponse -from . import params, schemas, crud, models -from core.dependencies import IdList -from typing import List +from core.dependencies import IdList +from . import params, schemas, crud, models +from utils.response import SuccessResponse, ErrorResponse + from fastapi import APIRouter, Depends, UploadFile, Form from apps.vadmin.auth.utils.current import FullAdminAuth from apps.vadmin.auth.utils.validation.auth import Auth @@ -60,12 +60,12 @@ async def del_project( return SuccessResponse(msg="删除成功") -@app.get("/label/{pro_id}", summary="查询标签列表") +@app.get("/label/{project_id}", summary="查询标签列表") async def label_list( - pro_id: int, + project_id: int, auth: Auth = Depends(FullAdminAuth()) ): - result = await crud.ProjectLabelDal(auth.db).get_datas(v_where=[models.ProjectLabel.project_id == pro_id], + result = await crud.ProjectLabelDal(auth.db).get_datas(v_where=[models.ProjectLabel.project_id == project_id], v_schema=schemas.ProjectLabel) return SuccessResponse(data=result) @@ -101,8 +101,7 @@ async def delete_label( ): # 删除标签信息，然后删除图片标签关联表 await crud.ProjectLabelDal(auth.db).delete_datas(label_ids.ids) - for label_id in label_ids.ids: - await crud.ProjectImgLabelDal(auth.db).delete_datas(label_id=label_id) + await crud.ProjectImgLabelDal(auth.db).del_img_label(label_ids.ids) return SuccessResponse(msg="删除成功") @@ -111,7 +110,7 @@ async def project_pager( param: params.ProjectImageParams = Depends(), auth: Auth = Depends(FullAdminAuth()) ): - if param.limit: + if param.limit > 0: # 分页查询，关联一个标签数量 datas, count = await crud.ProjectImageDal(auth.db).img_page(param) return SuccessResponse(data=datas, count=count) @@ -124,7 +123,7 @@ async def project_pager( @app.post("/img", summary="上传图片") async def up_img( project_id: int = Form(...), - files: List[UploadFile] = Form(...), + files: list[UploadFile] = Form(...), img_type: str = Form(...), auth: Auth = Depends(FullAdminAuth()) ): @@ -155,8 +154,10 @@ async def add_img_label( return SuccessResponse(msg="保存成功") - - - - - +@app.get("/img_label/{image_id}", summary="获取图片的标注信息") +async def get_img_label( + image_id: int, + auth: Auth = Depends(FullAdminAuth()) +): + leafer = await crud.ProjectImgLeaferDal(auth.db).get_leafer(image_id) + return SuccessResponse(data=leafer) \ No newline at end of file diff --git a/apps/vadmin/auth/crud.py b/apps/vadmin/auth/crud.py index e74b28d..53bb278 100644 --- a/apps/vadmin/auth/crud.py +++ b/apps/vadmin/auth/crud.py @@ -18,7 +18,7 @@ from sqlalchemy import select, false, and_ from core.crud import DalBase from sqlalchemy.ext.asyncio import AsyncSession from core.validator import vali_telephone -from utils.huawei_obs import ObsClient +from utils.huawei_obs import MyObs from utils.excel.import_manage import ImportManage, FieldType from utils.excel.write_xlsx import WriteXlsx from utils.send_email import EmailSender @@ -221,8 +221,6 @@ class UserDal(DalBase): value = getattr(user, field, "") if field == "is_active": value = "可用" if value else "停用" - elif field == "is_staff": - value = "是" if value else "否" elif field == "gender": result = list(filter(lambda i: i["value"] == value, options["gender_options"])) value = result[0]["label"] if result else "" @@ -397,7 +395,7 @@ class UserDal(DalBase): :param file: :return: """ - success, key, url = await ObsClient().put_object("avatar"+"/"+user.name+file.filename, file) + success, key, url = MyObs().put_object("avatar"+"/"+user.name+file.filename, file) user.avatar = url await self.flush(user) return url diff --git a/apps/vadmin/auth/models/user.py b/apps/vadmin/auth/models/user.py index 401729c..cd9cc6d 100644 --- a/apps/vadmin/auth/models/user.py +++ b/apps/vadmin/auth/models/user.py @@ -37,7 +37,6 @@ class VadminUser(BaseModel): ) last_ip: Mapped[str | None] = mapped_column(String(50), comment="最后一次登录IP") last_login: Mapped[datetime | None] = mapped_column(DateTime, comment="最近一次登录时间") - is_staff: Mapped[bool] = mapped_column(Boolean, default=False, comment="是否为工作人员") wx_server_openid: Mapped[str | None] = mapped_column(String(255), comment="服务端微信平台openid") is_wx_server_openid: Mapped[bool] = mapped_column(Boolean, default=False, comment="是否已有服务端微信平台openid") diff --git a/apps/vadmin/auth/params/user.py b/apps/vadmin/auth/params/user.py index f056da7..458f406 100644 --- a/apps/vadmin/auth/params/user.py +++ b/apps/vadmin/auth/params/user.py @@ -24,7 +24,6 @@ class UserParams(QueryParams): telephone: str | None = Query(None, title="手机号"), email: str | None = Query(None, title="邮箱"), is_active: bool | None = Query(None, title="是否可用"), - is_staff: bool | None = Query(None, title="是否为工作人员"), params: Paging = Depends() ): super().__init__(params) @@ -32,6 +31,5 @@ class UserParams(QueryParams): self.telephone = ("like", telephone) self.email = ("like", email) self.is_active = is_active - self.is_staff = is_staff diff --git a/apps/vadmin/auth/schemas/user.py b/apps/vadmin/auth/schemas/user.py index d1e7c74..63da857 100644 --- a/apps/vadmin/auth/schemas/user.py +++ b/apps/vadmin/auth/schemas/user.py @@ -21,7 +21,6 @@ class User(BaseModel): nickname: str | None = None avatar: str | None = None is_active: bool | None = True - is_staff: bool | None = True gender: str | None = "0" is_wx_server_openid: bool | None = False @@ -56,7 +55,6 @@ class UserUpdate(User): nickname: str | None = None avatar: str | None = None is_active: bool | None = True - is_staff: bool | None = False gender: str | None = "0" role_ids: list[int] = [] dept_ids: list[int] = [] diff --git a/apps/vadmin/auth/utils/current.py b/apps/vadmin/auth/utils/current.py index c384aad..35bc7b0 100644 --- a/apps/vadmin/auth/utils/current.py +++ b/apps/vadmin/auth/utils/current.py @@ -103,8 +103,7 @@ class FullAdminAuth(AuthValidation): telephone=telephone, password=password, v_return_none=True, - v_options=options, - is_staff=True + v_options=options ) result = await self.validate_user(request, user, db, is_all=False) permissions = self.get_user_permissions(user) diff --git a/apps/vadmin/auth/utils/login.py b/apps/vadmin/auth/utils/login.py index db6b287..bba5ef2 100644 --- a/apps/vadmin/auth/utils/login.py +++ b/apps/vadmin/auth/utils/login.py @@ -57,8 +57,6 @@ async def api_login_for_access_token( raise CustomException(status_code=error_code, code=error_code, msg="手机号或密码错误") if not user.is_active: raise CustomException(status_code=error_code, code=error_code, msg="此手机号已被冻结") - elif not user.is_staff: - raise CustomException(status_code=error_code, code=error_code, msg="此手机号无权限") access_token = LoginManage.create_token({"sub": user.telephone, "password": user.password}) record = LoginForm(platform='2', method='0', telephone=data.username, password=data.password) resp = {"access_token": access_token, "token_type": "bearer"} diff --git a/apps/vadmin/auth/utils/validation/login.py b/apps/vadmin/auth/utils/validation/login.py index 72c6cf6..424962c 100644 --- a/apps/vadmin/auth/utils/validation/login.py +++ b/apps/vadmin/auth/utils/validation/login.py @@ -80,7 +80,7 @@ class LoginValidation: await db.flush() elif not user.is_active: self.result.msg = "此手机号已被冻结！" - elif data.platform in ["0", "1"] and not user.is_staff: + elif data.platform in ["0", "1"]: self.result.msg = "此手机号无权限！" else: if not DEMO and count: diff --git a/apps/vadmin/system/views.py b/apps/vadmin/system/views.py index 34f3200..1fe09b0 100644 --- a/apps/vadmin/system/views.py +++ b/apps/vadmin/system/views.py @@ -6,19 +6,14 @@ # @desc : 主要接口文件 from redis.asyncio import Redis -from fastapi import APIRouter, Depends, Body, UploadFile, Form, Request +from fastapi import APIRouter, Depends, Body from motor.motor_asyncio import AsyncIOMotorDatabase -from sqlalchemy.ext.asyncio import AsyncSession -from application.settings import ALIYUN_OSS -from core.database import db_getter, redis_getter, mongo_getter -from utils.file.aliyun_oss import AliyunOSS, BucketConf -from utils.huawei_obs import ObsClient -from utils.file.file_manage import FileManage +from core.database import redis_getter, mongo_getter from utils.response import SuccessResponse, ErrorResponse from utils.sms.code import CodeSMS from . import schemas, crud from core.dependencies import IdList -from apps.vadmin.auth.utils.current import AllUserAuth, FullAdminAuth, OpenAuth +from apps.vadmin.auth.utils.current import AllUserAuth, OpenAuth from apps.vadmin.auth.utils.validation.auth import Auth from .params import DictTypeParams, DictDetailParams, TaskParams from apps.vadmin.auth import crud as vadmin_auth_crud diff --git a/core/crud.py b/core/crud.py index 17cbdd2..5d1a074 100644 --- a/core/crud.py +++ b/core/crud.py @@ -263,6 +263,16 @@ class DalBase: await self.db.execute(insert(self.model), datas) await self.db.flush() + async def create_models(self, datas: list[Any]) -> None: + """ + 批量创建数据 + SQLAlchemy 2.0 批量插入不支持 MySQL 返回值： + https://docs.sqlalchemy.org/en/20/orm/queryguide/dml.html#getting-new-objects-with-returning + :param datas: model数组 + """ + self.db.add_all(datas) + await self.db.flush() + async def put_data( self, data_id: int, diff --git a/utils/huawei_obs.py b/utils/huawei_obs.py index 2a454fd..a4aa0d8 100644 --- a/utils/huawei_obs.py +++ b/utils/huawei_obs.py @@ -5,28 +5,28 @@ from obs import DeleteObjectsRequest from obs import Object -class ObsClient: +class MyObs: - def __int__(self): + def __init__(self): self.obsClient = ObsClient( access_key_id=HUAWEI_OBS['AccessKeyID'], secret_access_key=HUAWEI_OBS['SecretAccessKey'], server=HUAWEI_OBS["server"]) - async def put_file(self, objectKey: str, file_path: str): - resp = await self.obsClient.putFile( + def put_file(self, object_key: str, file_path: str): + resp = self.obsClient.putFile( bucketName=HUAWEI_OBS["bucketName"], - objectKey=objectKey, + objectKey=object_key, file_path=file_path) if resp.status < 300: - print("objectKey", objectKey) + print("objectKey", object_key) print("url", resp.body.objectUrl) - return True, objectKey, resp.body.objectUrl + return True, object_key, resp.body.objectUrl else: return False, None, None - async def put_object(self, objectKey: str, file_content): - resp = await self.obsClient.put_object( + def put_object(self, objectKey: str, file_content): + resp = self.obsClient.put_object( bucketName=HUAWEI_OBS["bucketName"], objectKey=objectKey, file_cotent=file_content) @@ -37,14 +37,14 @@ class ObsClient: else: return False, None, None - async def del_objects(self, object_keys: []): + def del_objects(self, object_keys: []): objects = [] for object_key in object_keys: object_one = Object(key=object_key, versionId=None) objects.append(object_one) encoding_type = 'url' - resp = await self.obsClient.deleteObjects( + resp = self.obsClient.deleteObjects( bucketName=HUAWEI_OBS["bucketName"], deleteObjectsRequest=DeleteObjectsRequest(quiet=False, objects=objects, encoding_type=encoding_type)) if resp.status < 300: