Commit 11b2601e authored by qunfeng qiu's avatar qunfeng qiu
Browse files

Initial commit

parents
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import copy
import json
import uuid
from functools import wraps
from flask import current_app, request
from .cfginfohandle import CfgInfoHandle
from .globalconst import CookiesConstKey, FlaskConfigIDConst, RedisConnConst, RedisDbNumConst, GlobalConst, \
GlobalRetKeyConst, RedisInfoKeysConst
from .globalerror import CommonError
from .my_baseexception import MyBaseException
from .my_servicehandle import ServiceHandle
from .redishanlde import RedisHandle
class LoginHandle:
# 过滤规则
FILTER_RULE_NAME = "filter_name"
# 过滤具体值
FILTER_RULE_VALUE = "filter_val"
# 精确匹配
FILTER_EXACT = "filter_exact"
# 正则匹配
FILTER_REG = "filter_regular"
def __init__(self, flask_app=None, list_filter_url: list = None):
r"""
登录处理初始化
Args:
flask_app: flask app
list_filter_url: url过滤,不需要被登录检查的 list[dict]
"""
# 初始化,处理是否接入flask
self.list_filter_url = None
if list_filter_url is not None:
self.list_filter_url = list_filter_url
self.app = flask_app
if flask_app is not None:
self.init_app(flask_app)
def init_app(self, app):
r"""
初始化flask变量,走所有服务之前去验证登录
Args:
app:
Returns:
"""
# app.before_request(self.checklogin)
pass
def login(self, login_name: str = None, login_pass: str = None, login_code: str = None, tenant_id: str = None,
proc_name: str = "auth_userlogin_proc", login_srv_url: str = None, redis_info: dict = None) -> tuple:
R"""
Args:
login_name: 登录账号
login_pass: 登录密码
login_code: 登录验证码
tenant_id: 租户ID
proc_name: proc名称
login_srv_url: 登录url
redis_info: redis信息
Returns:
"""
# 登录验证
service_url = login_srv_url
trans_data = {
"arg_tenantid": tenant_id
, "arg_username": login_name
, "arg_userpass": login_pass
}
sh = ServiceHandle()
ret_json: dict = sh.call_proc(service_url=service_url, proc_name=proc_name, arg_dict=trans_data)
# 返回失败
if sh.check_is_success(ret_json) is False or GlobalRetKeyConst.DATAROWS not in ret_json or len(
ret_json[GlobalRetKeyConst.DATAROWS]) == 0:
return ret_json, None
user_info: dict = ret_json[GlobalRetKeyConst.DATAROWS][0]
jsessionid = str(uuid.uuid1()).replace('-', '')
dict_kv = {
jsessionid: json.dumps(user_info, ensure_ascii=False)
}
# 校验成功写入缓存
rh: RedisHandle = current_app.config[FlaskConfigIDConst.REDIS_CLASS_ID]
rh.set_vals(dict_redisinfo=redis_info, dict_kv=dict_kv)
return ret_json, jsessionid
def check_login(func):
r"""
装饰器检测是否登录
Returns:
"""
@wraps(func)
def wrapper(*args, **kwargs):
if_login = False
# 检查cookies是否为null 或者 cookies中的sessionid的值是否在redis中存在
if request.cookies.keys() is not None and CookiesConstKey.LOGIN_SESSTION_KEY in request.cookies.keys():
if_login = True
rh: RedisHandle = current_app.config[FlaskConfigIDConst.REDIS_CLASS_ID]
cfg_handle: CfgInfoHandle = current_app.config[FlaskConfigIDConst.INIT_CONFIG_ID]
dict_conn: dict = copy.copy(cfg_handle.get_redis_info()[RedisInfoKeysConst.REDIS_SLAVE_KEY])
dict_conn[RedisConnConst.CONN_DB] = RedisDbNumConst.LOGIN_INFO
csm_id = request.cookies[CookiesConstKey.LOGIN_SESSTION_KEY]
dict_key = {
csm_id: csm_id
}
ret_dict = rh.get_vals(dict_key, dict_conn)
if csm_id not in ret_dict:
if_login = False
if if_login is True:
return func(*args, **kwargs)
else:
return json.dumps(MyBaseException.format_to_standard_dict(ret_code=GlobalConst.RETCODE_COMMON_ERROR,
ret_val=CommonError.ERROR_USER_UNLOGIN
), ensure_ascii=False).encode('utf-8')
return wrapper
# venv 安装
建议使用 venv 虚拟环境 使用pycharm自动生成venv
或者在项目文件夹下执行下面的语句后再在pycharm中设置python解释器为虚拟环境中的解释器
```shell
# 创建虚拟环境
python3 -m venv venv
# 激活虚拟环境
source venv/Scripts/activate
# 或者在windows 下直接使用 venv\Scripts\activate 激活
```
配置好虚拟环境后,再进行依赖包安装
# 依赖包安装
```shell
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 simplejson==3.17.0
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 Flask==1.1.2
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 pymysql==0.10.1
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 redis==3.5.3
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 pytz==2021.1
# aes加密使用的库
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 pycryptodome==3.9.9
# yaml文件
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 pyyaml==5.3.1
# rsa解密
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 rsa==4.6
# 数据库连接池
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 DBUtils==2.0
# COS操作
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 boto==2.49.0
# rarfile操作
pip3 install -i http://10.126.154.1:18083/repository/swrdcucc-group/simple --trusted-host 10.126.154.1 rarfile==4.0
# 卸载模块
pip3 uninstall cucc_common_pkg
```
# 发布
```
pip3 install --upgrade setuptools wheel twine
# 如有必要, 先将原先编译生成的目录删掉, 例如 build,dist目录
# setup.py文件同目录命令行下运行
python setup.py sdist bdist_wheel
# 检查打包的文件是否正常
python setup.py install # 安装
# 按照使用方式导入测试,没问题后继续
# 上传
twine upload dist/* --repository-url http://10.126.154.1:18083/repository/swrdcucc-hosted/ -u {wenhx8} -p {dwp}
```
# 项目调试
## 配置环境变量
```
PYTHONUNBUFFERED=1
ENV_APP_CONFIG={"db_config":{"host":"sky-mysql","port":3306,"user":"","passwd":"","database":"","charset":"utf8mb4"},"master_redis_config":{"host":"csm-redis-master.default.svc","port":6379,"password":"","db":"5"},"slave_redis_config":{"host":"csm-redis-slave.default.svc","port":6379,"password":"","db":"5"},"is_debug":false,"is_send_admin_notice":false,"is_send_user_notice":false,"send_email_config":{"mail_host":"hq.smtp.chinaunicom.cn","mail_from_account":"hqs-ioa-cusri@chinaunicom.cn","mail_from_account_dwp":""},"send_sms_config":{"api_server":"https://sms.tg.unicom.local/sms/message/send","sys_name":"","sys_token":""},"admin_list":["ludq1@chinaunicom.cn"],"runtime_env":"DEV","aes_mysql_key":"","rsa_private_key":"","sso_api":"https://sso.dev.tg.unicom.local/sso/v1","iam_api":"https://iam.dev.tg.unicom.local/iam/v2","configcenter_api":"https://configcenter.dev.tg.unicom.local/configcenter/v1","product_vpc_api":"https://vpc.console.dev.tg.unicom.local","product_clb_api":"https://vpc.console.dev.tg.unicom.local","product_cke_api":"https://cke.console.dev.tg.unicom.local","product_csm_api":"https://csm.console.dev.tg.unicom.local","product_ccr_api":"https://ccr.console.dev.tg.unicom.local","product_rds_api":"https://rds.console.dev.tg.unicom.local","product_redis_api":"https://redis.console.tg.unicom.local","product_kafka_api":"https://kafka.console.dev.tg.unicom.local","tianti_api":"https://tianti.dev.tg.unicom.local","ide_apiserver":"","ide_apiserver_secret":"disable","ide_image":""}
ENV_APP_CONFIG_INIT_STYLE=file
ENV_APP_CONFIG_FILE_PATH=C:\Users\supershll\Desktop\app_config.json
```
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import base64
from typing import Optional
from Crypto.Cipher import AES
class AesUtilsForMysql:
r"""
Mysql AES加密 utils类,不依赖任何其他应用基础类
"""
def __init__(self, key: str = None):
r"""
初始化
:param key
"""
self.key: str = key
if self.key:
self.real_aes_key: Optional[bytes] = self.gen_aes_key_by_mysql_key(self.key)
else:
self.real_aes_key: Optional[bytes] = None
def pkcs7padding_to_encrypt_content(self, to_encrypt_content: str) -> str:
r"""
明文使用PKCS7填充
最终调用AES加密方法时,传入的是一个byte数组,要求是16的整数倍,因此需要对明文进行处理
:param to_encrypt_content
:return
"""
bytes_length = len(bytes(to_encrypt_content, encoding='utf-8'))
# tips:utf-8编码时,英文占1个byte,而中文占3个byte
padding_times = 16 - bytes_length % 16
# tips:chr(padding)看与其它语言的约定,有的会使用'\0'
padding_text = chr(padding_times) * padding_times
return to_encrypt_content + padding_text
def pkcs7unpadding_decrypted_content(self, decrypted_content: str) -> str:
r"""
处理使用PKCS7填充过的数据
:param decrypted_content
:return
"""
length = len(decrypted_content)
unpadding = ord(decrypted_content[length - 1])
return decrypted_content[0:length - unpadding]
def gen_aes_key_by_mysql_key(self, key: str = None) -> bytes:
r"""
生成mysql函数中使用的key实际对应的aes_key
:param key
"""
real_key = key if key else self.key
final_key = bytearray(16)
for i, c in enumerate(real_key):
final_key[i % 16] ^= ord(real_key[i])
return bytes(final_key)
def encrypt(self, to_encrypt_content: str, key: str = None) -> str:
r"""
mysql AES加密
模式 AES.MODE_ECB
填充pkcs7
加密后使用 base64 编码
:param to_encrypt_content
:param key
:return
"""
real_aes_key = self.gen_aes_key_by_mysql_key(key) if key else self.real_aes_key
cipher = AES.new(real_aes_key, AES.MODE_ECB)
# 将内容编码
to_encrypt_content = self.pkcs7padding_to_encrypt_content(to_encrypt_content)
byte_content = to_encrypt_content.encode('utf-8')
byte_encrypted = cipher.encrypt(byte_content)
# base64 编码
byte_base64 = base64.b64encode(byte_encrypted)
return str(byte_base64, 'utf-8')
def decrypt(self, to_decrypt_content: str, key: str = None):
r"""
mysql AES解密
模式 AES.MODE_ECB
去填充pkcs7
解密前使用 base64 解码
:param to_decrypt_content
:param key
:return
"""
real_aes_key = self.gen_aes_key_by_mysql_key(key) if key else self.real_aes_key
cipher = AES.new(real_aes_key, AES.MODE_ECB)
# 先将内容解码
byte_base64 = to_decrypt_content.encode('utf-8')
byte_encrypted = base64.b64decode(byte_base64)
# 解码
result = cipher.decrypt(byte_encrypted)
result = str(result, 'utf-8')
# 去除填充内容
result = self.pkcs7unpadding_decrypted_content(result)
return result
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from .app_response import AppResponse
from .base_const import ConstResponseCode
from ..globalutility import Utility
class AppException(Exception):
"""
自定义的异常类, 表示可控的异常,不依赖任何其他应用基础类
"""
code: str = None
message: str = None
_status_code: int = None
def __init__(self, code: str = None, message: str = None, status_code: int = 200):
r"""
初始化异常
:param code
:param message
:param status_code
"""
self.code: str = code or ConstResponseCode.CODE_SYS_ERROR
self.message: str = message or ConstResponseCode.gen_msg_for_code(self.code)
self.status_code = status_code
self.my_response = AppResponse(code=self.code, message=self.message, status_code=status_code)
def __str__(self):
r"""
字符串表示, 使用json字符串表示
Returns:
"""
return str(self.my_response)
def __repr__(self):
r"""
字符串表示,使用 __str__ , 不会返回None,至少时 EMPTY字符串
Returns:
"""
return repr(self.my_response)
def gen_err(self) -> dict:
r"""
生成带有 code,message的标准dict
Returns:
"""
return self.my_response.gen_dict()
@property
def status_code(self) -> int:
r"""
status_code 属性
:return:
"""
if self._status_code is None or self._status_code < 1:
self._status_code = 200
return self._status_code
@status_code.setter
def status_code(self, status_code: int):
r"""
status_code 属性
:param status_code:
:return:
"""
self._status_code = status_code
class AppRuntimeException(Exception):
"""
自定义的异常类, 区分其他的异常
还用于返回给前端基础的异常信息,和在后端打印的详细异常信息
"""
message: str = None
detail: str = None
code: str = None
def __init__(self, message: str, detail: str = None, code: str = None):
r"""
初始化
:param message
:param detail
"""
self.message: str = message
self.detail: str = detail
self.code: str = code
def __str__(self):
r"""
字符串表示
Returns:
"""
return Utility.dict2jsonstr({"message": self.message, "detail": self.detail, "code": self.code})
def __repr__(self):
r"""
字符串表示
Returns:
"""
return self.__str__()
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from .base_const import ConstResponseCode
from ..globalutility import Utility
class AppResponse:
r"""
标准返回数据的封装,包含 code,msg,data等key的标准dict的json表示,不依赖任何其他应用基础类
"""
code: str = None
message: str = None
data = None
other_data: dict = None
_status_code: int = None
def __init__(
self,
code: str = None, message: str = None, data=None, other_data: dict = None,
status_code: int = 200,
):
r"""
初始化,默认为OK的返回
:param code: 默认为 SysError
:param message: 默认为 系统错误
:param data:
:param other_data:
:param status_code:
"""
self.code: str = code or ConstResponseCode.CODE_OK
self.message: str = message or ConstResponseCode.gen_msg_for_code(self.code)
self.other_data = other_data if other_data is not None else dict()
self.data = data
self.status_code = status_code
@classmethod
def from_dict(
cls,
source_dict: dict,
key_for_msg: str = "message",
key_for_data: str = "data",
key_for_code: str = "code",
key_for_status_code: str = "status_code",
):
r"""
根据 source_dict(包含 code 和 nessage key的dict) 生成 AppResponse
:param source_dict:
:param key_for_code:
:param key_for_data:
:param key_for_msg:
:param key_for_status_code:
:return:
"""
if not source_dict:
raise RuntimeError("AppResponse解析dict对象出错:dict对象为空")
result: AppResponse = AppResponse()
key_for_code = key_for_code or "code"
tmp_code = source_dict.get(key_for_code)
if not tmp_code:
raise RuntimeError(Utility.join_str("AppResponse解析dict对象出错:dict对象的", key_for_code, "不合规"))
result.code = str(tmp_code)
key_for_msg = key_for_msg or "message"
tmp_msg = source_dict.get(key_for_msg) or ""
result.message = tmp_msg
key_for_data = key_for_data or "data"
tmp_data = source_dict.get(key_for_data)
result.data = tmp_data
key_for_status_code = key_for_status_code or "status_code"
tmp_data = source_dict.get(key_for_status_code)
result.status_code = tmp_data
for tmp_key, tmp_value in source_dict.items():
if tmp_key not in {key_for_code, key_for_msg, key_for_data, key_for_status_code}:
continue
result.other_data[tmp_key] = tmp_value
return result
def __str__(self):
r"""
字符串表示, 使用json字符串表示
Returns:
"""
return Utility.dict2jsonstr(self.gen_dict())
def __repr__(self):
r"""
字符串表示,使用 __str__ , 不会返回None,至少时 EMPTY字符串
Returns:
"""
return self.__str__()
def gen_dict(self) -> dict:
r"""
生成带有 code,message的标准dict
Returns:
"""
ret_dict = {"code": self.code, "message": self.message, "status_code": self.status_code}
if self.other_data:
for key, value in self.other_data.items():
ret_dict[key] = value
if self.data is not None:
ret_dict['data'] = self.data
return ret_dict
@property
def status_code(self) -> int:
r"""
status_code 属性
:return:
"""
if self._status_code is None or self._status_code < 1:
self._status_code = 200
return self._status_code
@status_code.setter
def status_code(self, status_code: int):
r"""
status_code 属性
:param status_code:
:return:
"""
self._status_code = status_code
def is_ok(self) -> bool:
r"""
验证code是否等于OK
"""
return self.code == ConstResponseCode.CODE_OK
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
class ConstBaseApp(object):
COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH = "/tmp/common_app_config.conf"
class ConstResponseCode(object):
CODE_OK: str = 'OK'
CODE_SYS_ERROR: str = 'SysError'
CODE_MISSING_PARAMETER: str = 'MissingParameter'
CODE_STATUS_ERROR: str = 'StatusError'
CODE_INVALID_ACTION: str = 'InvalidAction'
CODE_AUTH_FAILURE: str = 'AuthFailure'
CODE_UNAUTHORIZE_OPERATION: str = 'UnauthorizedOperation'
CODE_K8S_ERROR: str = 'K8sError'
CODE_MAP: dict = {
CODE_OK: "成功",
CODE_SYS_ERROR: "系统错误",
CODE_MISSING_PARAMETER: "参数错误",
CODE_STATUS_ERROR: "实例状态不允许当前的操作",
CODE_INVALID_ACTION: "请求的URI地址不存在",
CODE_AUTH_FAILURE: "登陆已超期",
CODE_UNAUTHORIZE_OPERATION: "API访问未授权",
CODE_K8S_ERROR: "K8S调用错误",
}
@classmethod
def gen_msg_for_code(cls, code: str) -> str:
return cls.CODE_MAP.get(code, code)
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import logging
from typing import Optional
from pymysql.connections import Connection
from pymysql.cursors import DictCursor
from .app_exception import AppRuntimeException
from .common_app_config import CommonAppConfig
from ..globalutility import Utility
from ..my_stringutils import MyStringUtils
class QueryResultWithFoundRows:
result_list: Optional[list] = None
found_rows: Optional[int] = None
def __init__(self, **kwargs):
self.result_list = kwargs.get("result_list")
self.found_rows = kwargs.get("found_rows")
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
class DbBehaviourAfterExec:
r"""
数据库语句执行完成后的连接操作行为,不依赖任何其他应用基础类
默认 commit_after_exec 和 close_conn_after_exec 为 False
close_conn_when_exception 和 rollback_when_exception 为 True ,commit_when_exception 为 False
rollback_when_exception 的优先级比 commit_when_exception 高,
当 close_conn_when_exception 为True时,如果 rollback_when_exception 和 commit_when_exception 都为False,
则执行 rollback 操作
"""
commit_after_exec: bool = True
close_conn_after_exec: bool = True
close_conn_when_exception: bool = True
rollback_when_exception: bool = True
commit_when_exception: bool = False
def __init__(self, commit_after_exec: bool = True, close_conn_after_exec: bool = True,
close_conn_when_exception: bool = True, rollback_when_exception: bool = True,
commit_when_exception: bool = False):
r"""
初始化数据库行为
:param commit_after_exec:
:param close_conn_after_exec:
:param close_conn_when_exception:
:param rollback_when_exception:
:param commit_when_exception:
"""
self.commit_after_exec = commit_after_exec
self.close_conn_after_exec = close_conn_after_exec
self.close_conn_when_exception = close_conn_when_exception
self.rollback_when_exception = rollback_when_exception
self.commit_when_exception = commit_when_exception
@classmethod
def gen_no_commit_close_behaviour(cls):
r"""
生成一个不commit,不close,异常时rollback、close的数据库行为
:return:
"""
return DbBehaviourAfterExec(
commit_after_exec=False,
close_conn_after_exec=False,
close_conn_when_exception=True,
rollback_when_exception=True,
commit_when_exception=False)
class BaseDbService:
r"""
基础 dbService类,定义一些基础函数,不依赖任何其他应用基础类
"""
def __init__(self, logger: logging.Logger = None):
self.logger = logger or CommonAppConfig().common_logger
def join_str(self, *to_join_list: str, separator_str: str = '', wrapper_str: str = None,
ignore_none: bool = False) -> str:
r"""
连接 list, 当值为 None 时当做 null字符串 添加,但是不追加 wrapper_str
:param to_join_list
:param separator_str 默认使用空字符串
:param wrapper_str
:param ignore_none 默认为True
:return
"""
return Utility.list_join_to_str(to_join_list, separator_str=separator_str, wrapper_str=wrapper_str,
ignore_none=ignore_none)
def do_filter_mysql_param_with_single_quotes(self, to_filter_param: str, append_fix: str = "'") -> str:
r"""
mysql防注入使用,用于过滤(替换)拼接字符串中的参数,当拼接`key`='value'时,对value中的特殊字符进行替换
:param to_filter_param
:param append_fix 默认为单引号
:return
"""
return Utility.do_filter_mysql_param(to_filter_param=to_filter_param, append_fix=append_fix)
def do_filter_mysql_param_with_single_quotes_for_fuzzy_search(self, to_filter_param: str,
append_fix: str = "'") -> str:
r"""
mysql防注入使用,用于过滤(替换)拼接字符串中的参数,当拼接`key`='value'时,对value中的特殊字符进行替换
:param to_filter_param
:param append_fix 默认为单引号
:return
"""
return Utility.do_filter_mysql_param_with_single_quotes_for_fuzzy_search(to_filter_param=to_filter_param,
append_fix=append_fix)
def do_filter_mysql_param_list_with_single_quotes(
self,
to_filter_param_list: list,
append_fix: str = "'"
) -> Optional[list]:
r"""
mysql防注入使用,用于过滤(替换)拼接字符串中的参数,当拼接`key`='value'时,对value中的特殊字符进行替换
Utility.do_filter_mysql_param_list_with_single_quotes函数有bug,所以自己实现一下
:param to_filter_param_list
:param append_fix 默认为单引号
:return
"""
if to_filter_param_list is None:
return None
return [self.do_filter_mysql_param_with_single_quotes(tmp_key, append_fix=append_fix) for tmp_key in
to_filter_param_list]
def gen_default_db_behaviour_after_exec(self):
r"""
生成默认的 db_behaviour_after_exec
默认行为为 正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:return:
"""
return DbBehaviourAfterExec(
commit_after_exec=True, close_conn_after_exec=True,
close_conn_when_exception=True,
rollback_when_exception=True,
commit_when_exception=False)
def commit(self, conn: Connection, close_conn_after_exec: bool = True,
db_behaviour_after_exec: DbBehaviourAfterExec = None) -> bool:
r"""
执行commit操作
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param close_conn_after_exec
:param db_behaviour_after_exec
:return
"""
if conn is None:
return True
conn.commit()
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
if close_conn_after_exec or (db_behaviour_after_exec and db_behaviour_after_exec.close_conn_after_exec):
conn.close()
return True
def rollback(self, conn: Connection, close_conn_after_exec: bool = True,
db_behaviour_after_exec: DbBehaviourAfterExec = None) -> bool:
r"""
执行rollback操作
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param close_conn_after_exec
:param db_behaviour_after_exec
:return
"""
if conn is None:
return True
conn.rollback()
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
if close_conn_after_exec or (db_behaviour_after_exec and db_behaviour_after_exec.close_conn_after_exec):
conn.close()
return True
def close_conn(self, conn: Connection):
r"""
关闭连接
:param conn
"""
if conn is None:
return
conn.close()
def do_after_exec(self, conn: Connection, result, encountered_exception,
db_behaviour_after_exec: DbBehaviourAfterExec):
r"""
数据库执行完毕后的行为封装
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param result
:param encountered_exception
:param db_behaviour_after_exec
:return:
"""
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
if not encountered_exception:
return self.do_after_exec_without_exception(conn, result, db_behaviour_after_exec)
else:
return self.do_after_exec_with_exception(conn, encountered_exception, db_behaviour_after_exec)
def do_after_exec_without_exception(self, conn: Connection, result, db_behaviour_after_exec: DbBehaviourAfterExec):
r"""
数据库执行完毕后的行为封装
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param result
:param db_behaviour_after_exec
:return:
"""
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
close_conn_after_exec = True if db_behaviour_after_exec.close_conn_after_exec else False
commit_after_exec = True if db_behaviour_after_exec and db_behaviour_after_exec.commit_after_exec else False
if commit_after_exec and conn is not None:
conn.commit()
if close_conn_after_exec and conn is not None:
conn.close()
return result
def do_after_exec_with_exception(self, conn: Connection, encountered_exception,
db_behaviour_after_exec: DbBehaviourAfterExec):
r"""
数据库执行完毕后的行为封装
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param encountered_exception
:param db_behaviour_after_exec
:return:
"""
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
close_conn_when_exception = True if db_behaviour_after_exec.close_conn_when_exception else False
rollback_when_exception = True if db_behaviour_after_exec.rollback_when_exception else False
commit_when_exception = True if db_behaviour_after_exec.commit_when_exception else False
if close_conn_when_exception and not rollback_when_exception and not commit_when_exception:
# 当异常时关闭连接 但是 rollback_when_exception 和 commit_when_exception 都为False,则默认使用 rollback
rollback_when_exception = True
if rollback_when_exception and conn is not None:
conn.rollback()
elif commit_when_exception and conn is not None:
conn.commit()
if close_conn_when_exception and conn is not None:
conn.close()
raise encountered_exception
def querydb(
self, conn: Connection, query_str: str, factory_method=None,
db_behaviour_after_exec: DbBehaviourAfterExec = None
) -> list:
r"""
查询数据库
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param factory_method
:param db_behaviour_after_exec
:return
"""
if conn is None:
raise AppRuntimeException(message="未提供conn", detail="执行数据库操作时,传入的conn为None")
encountered_exception: Optional[BaseException] = None
result: Optional[list] = None
try:
self.logger.debug(f"执行sql语句:{query_str}")
with conn.cursor(cursor=DictCursor) as cursor:
try:
cursor.execute(query_str)
result = cursor.fetchall() or list()
finally:
cursor.close()
self.logger.debug(f"执行sql语句:{query_str} 返回结果: {result}")
except BaseException as e:
encountered_exception = e
self.logger.debug(f"执行sql语句:{query_str} 发生异常: {e}")
if not encountered_exception and factory_method is not None and result:
try:
result = [factory_method(tmpDict) for tmpDict in result]
except BaseException as e:
encountered_exception = e
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
return self.do_after_exec(conn, result, encountered_exception, db_behaviour_after_exec)
def querycount_db(self, conn: Connection, query_str: str,
db_behaviour_after_exec: DbBehaviourAfterExec = None) -> int:
r"""
查询数据库,执行count数据库查询,直接得到返回的整数
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param db_behaviour_after_exec
:return
"""
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
ret_list = self.querydb(conn, query_str, db_behaviour_after_exec=db_behaviour_after_exec)
if not ret_list:
raise ValueError('数据库查询语句返回结果集为空')
return int(list(ret_list[0].values())[0])
def querydb_with_single_column_and_line(self, conn: Connection, query_str: str,
db_behaviour_after_exec: DbBehaviourAfterExec = None) -> Optional[str]:
r"""
查询数据库,执行返回单列单行的数据,并获取字符串,查询的数据不存在时,返回None
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param db_behaviour_after_exec
:return
"""
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
ret_list = self.querydb(conn, query_str, db_behaviour_after_exec=db_behaviour_after_exec)
if not ret_list:
return None
return MyStringUtils.to_str(list(ret_list[0].values())[0])
def querydb_with_sql_calc_found_rows(
self, conn: Connection, query_str: str, factory_method=None,
db_behaviour_after_exec: DbBehaviourAfterExec = None
) -> QueryResultWithFoundRows:
r"""
查询数据库,为带有 SQL_CALC_FOUND_ROWS 的sql语句返回 FOUND_ROWS()
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param factory_method
:param db_behaviour_after_exec
:return
"""
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
tmp_db_behaviour_after_exec = DbBehaviourAfterExec.gen_no_commit_close_behaviour()
tmp_db_behaviour_after_exec.close_conn_when_exception = db_behaviour_after_exec.close_conn_when_exception
tmp_db_behaviour_after_exec.commit_when_exception = db_behaviour_after_exec.commit_when_exception
tmp_db_behaviour_after_exec.rollback_when_exception = db_behaviour_after_exec.rollback_when_exception
ret_list = self.querydb(conn, query_str, factory_method=factory_method,
db_behaviour_after_exec=tmp_db_behaviour_after_exec)
found_rows = self.querydb_with_single_column_and_line(
conn=conn,
query_str="select FOUND_ROWS() as found_rows",
db_behaviour_after_exec=db_behaviour_after_exec
)
found_rows = int(found_rows)
return QueryResultWithFoundRows(result_list=ret_list, found_rows=found_rows)
def operate_db(self, conn: Connection, *operate_sql_list: str, sql_list: list = None,
db_behaviour_after_exec: DbBehaviourAfterExec = None) -> int:
r"""
操作数据库,返回受影响的行数, 可以有两种提供操作语句的方式,或者直接执行多条语句,或者使用 sql_list 提供操作语句列表
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param operate_sql_list
:param sql_list
:param db_behaviour_after_exec
:return
"""
if conn is None:
raise AppRuntimeException(message="未提供conn", detail="执行数据库操作时,传入的conn为None")
encountered_exception: Optional[BaseException] = None
result: int = 0
try:
# 先生成列表
real_sql_list = []
if operate_sql_list:
real_sql_list += operate_sql_list
elif sql_list:
real_sql_list += sql_list
else:
raise ValueError('未提供sql语句')
with conn.cursor(cursor=DictCursor) as cursor:
try:
for tmp_sql_str in real_sql_list:
self.logger.debug(f"执行sql语句:{tmp_sql_str}")
result += cursor.execute(tmp_sql_str)
self.logger.debug(f"执行sql语句:{tmp_sql_str} 返回结果: {result}")
finally:
cursor.close()
except BaseException as e:
encountered_exception = e
if db_behaviour_after_exec is None:
db_behaviour_after_exec = self.gen_default_db_behaviour_after_exec()
return self.do_after_exec(conn, result, encountered_exception, db_behaviour_after_exec)
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import random
from typing import List, Dict
import redis
from redis import StrictRedis
from redis.sentinel import Sentinel
from .common_app_config import CommonAppConfig
from .utils_base import UtilityBaseV2
class BaseRedisService:
r"""
redis操作基类
"""
# 当调用jedis API时发生错误后重试的次数, 默认为1
retry_times_when_exception: int = 1
app_utils: UtilityBaseV2 = None
need_decode_responses: bool = True
r"""
获取到redis函数结果后是否需要解码
如果conn的初始化中包含 decode_responses=True,则应该设置此属性为False,
即获取到的redis函数结果已经是解码后的数据了,不需要再次解码
"""
encoding_used_for_decode_response: str = "utf-8"
def __init__(self, *args, **kwargs):
r"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key logger: 用于打印日志
"""
retry_times_when_exception = kwargs.get("retry_times_when_exception")
app_utils = kwargs.get("app_utils")
self.retry_times_when_exception = retry_times_when_exception if retry_times_when_exception > 0 else 1
self.app_utils = app_utils if app_utils else UtilityBaseV2(
logger=kwargs.get("logger") or CommonAppConfig().common_logger,
)
self.logger = kwargs.get("logger") or CommonAppConfig().common_logger
def set(self, name: str, value: str, ex: int = None, px: int = None, nx: bool = False, xx: bool = False) -> bool:
r"""
Set the value at key ``name`` to ``value`` , 返回 bool 表示是否成功执行
:param name
:param value
:param ex sets an expire flag on key ``name`` for ``ex`` seconds.
:param px sets an expire flag on key ``name`` for ``px`` milliseconds.
:param nx if set to True, set the value at key ``name`` to ``value`` only if it does not exist.
:param xx if set to True, set the value at key ``name`` to ``value`` only if it already exists.
"""
def operation_func():
return self.get_master_conn().set(name=name, value=value, ex=ex, px=px, nx=nx, xx=xx)
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def get(self, name: str) -> str:
r"""
Return the value at key ``name``, or None if the key doesn't exist,返回 utf-8解码的字符串
:param name
:return
"""
def operation_func():
result = self.get_slave_conn().get(name=name)
if self.need_decode_responses and result is not None:
result = str(result, self.encoding_used_for_decode_response)
return result
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def mset(self, name_value_dict: dict) -> bool:
r"""
Sets key/values based on a mapping. Mapping is a dictionary of
key/value pairs. Both keys and values should be strings or types that
can be cast to a string via str() ,
返回 bool 表示是否成功执行
:param name_value_dict
:return
"""
def operation_func():
return self.get_master_conn().mset(mapping=name_value_dict)
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def mget(self, *to_get_names: str) -> list:
r"""
Returns a list of values ordered identically to ``keys``,
返回 utf-8解码的字符串list
:param to_get_names
:return
"""
def operation_func():
result = self.get_slave_conn().mget(keys=to_get_names)
if self.need_decode_responses and result is not None:
result = [
str(tmp_byte_value, self.encoding_used_for_decode_response) if tmp_byte_value is not None else None
for
tmp_byte_value in
result]
return result
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def hset(self, name: str, key: str, value: str) -> int:
r"""
Set ``key`` to ``value`` within hash ``name``
Returns 1 if HSET created a new field, otherwise 0, 返回 int 表示是否成功插入的key的数量
:param name
:param key
:param value
"""
def operation_func():
return self.get_master_conn().hset(name=name, key=key, value=value)
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def hget(self, name: str, key: str) -> str:
r"""
Returns a list of values ordered identically to ``keys``,返回 utf-8解码的字符串list
:param name
:param key
"""
def operation_func():
result = self.get_slave_conn().hget(name=name, key=key)
if self.need_decode_responses and result is not None:
result = str(result, self.encoding_used_for_decode_response)
return result
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def hmset(self, name: str, key_value_dict: dict) -> bool:
r"""
Set key to value within hash ``name`` for each corresponding
key and value from the ``mapping`` dict. ,
返回 bool 表示是否成功执行
:param name
:param key_value_dict
"""
def operation_func():
return self.get_master_conn().hmset(name=name, mapping=key_value_dict)
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def hmget(self, *to_get_keys: str, name: str) -> list:
r"""
Returns a list of values ordered identically to ``keys``,
返回 utf-8解码的字符串list
:param to_get_keys
:param name
"""
def operation_func():
result = self.get_slave_conn().hmget(name=name, keys=to_get_keys)
if self.need_decode_responses and result is not None:
result = [
str(tmp_byte_value, self.encoding_used_for_decode_response) if tmp_byte_value is not None else None
for
tmp_byte_value in
result]
return result
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def hgetall(self, name: str) -> dict:
r"""
Return a Python dict of the hash's name/value pairs ,
返回 utf-8解码的key,value字符串 的 dict
:param name
"""
def operation_func():
result = self.get_slave_conn().hgetall(name=name)
if self.need_decode_responses and result is not None:
result = {str(tmp_byte_key, self.encoding_used_for_decode_response): str(result[tmp_byte_key],
self.encoding_used_for_decode_response)
for
tmp_byte_key in
result.keys()}
return result
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def expire(self, name: str, time: int, use_second: bool = False, use_millisecond: bool = False) -> bool:
r"""
设置key的过期时间, 从现在开始算过多少秒或毫秒之后key会被删除
如果 use_second 为 True ,则 time 单位为秒的int或者 python timedelta object.
如果 use_millisecond 为 True , 则 time 单位为毫秒的int或者 python timedelta object.
use_second 和 use_millisecond 必须至少有一个为True
返回 bool 表示是否成功执行
:param name
:param time
:param use_millisecond
:param use_second
"""
def operation_func():
if use_second:
return self.get_master_conn().expire(name=name, time=time)
elif use_millisecond:
return self.get_master_conn().pexpire(name=name, time=time)
else:
raise RuntimeError('use_second 和 use_millisecond 必须至少有一个为True')
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def expireat(self, name: str, when: int, use_second: bool = False, use_millisecond: bool = False) -> bool:
r"""
设置key的过期时间, 在指定的时间点key会被删除
如果 use_second 为 True ,则 when 为 unix time or a Python datetime object
如果 use_millisecond 为 True , 则 when 为 unix time in milliseconds (unix time * 1000) or a Python datetime object.
use_second 和 use_millisecond 必须至少有一个为True
返回 bool 表示是否成功执行
:param name
:param when
:param use_second
:param use_millisecond
"""
def operation_func():
if use_second:
return self.get_master_conn().expireat(name=name, when=when)
elif use_millisecond:
return self.get_master_conn().pexpireat(name=name, when=when)
else:
raise RuntimeError('use_second 和 use_millisecond 必须至少有一个为True')
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def delete(self, *to_delete_names: str) -> int:
r"""
Delete one or more keys specified by ``names``
返回 int 表示是否成功删除的key的数量
:param to_delete_names
"""
def operation_func():
return self.get_master_conn().delete(*to_delete_names)
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def keys(self, pattern: str) -> list:
r"""
Returns a list of keys matching ``pattern``,
返回 utf-8解码的字符串list
:param pattern
"""
def operation_func():
result = self.get_slave_conn().keys(pattern=pattern)
if self.need_decode_responses and result is not None:
result = [str(tmp_byte_value, self.encoding_used_for_decode_response) for tmp_byte_value in result]
return result
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def delete_by_pattern(self, pattern: str) -> int:
r"""
按 pattern 删除key,
返回 int 表示是否成功删除的key的数量
:param pattern
"""
def operation_func():
key_list = self.get_slave_conn().keys(pattern=pattern)
if key_list:
return self.get_master_conn().delete(*key_list)
else:
return 0
return self.app_utils.common_operation(retry_times_when_exception=self.retry_times_when_exception,
operation_func=operation_func,
on_operation_failed=self.on_operation_failed,
on_operation_completed=self.on_operation_completed)
def on_operation_failed(self):
r"""
当redis操作失败时的动作,一般是重建连接,或者等待几秒之类的操作
:return:
"""
pass
def on_operation_completed(self):
r"""
当redis操作完成时的动作,一般是什么都不做
:return:
"""
pass
def get_master_conn(self) -> StrictRedis:
r"""
获取master连接
:return:
"""
raise RuntimeError("Not implemented")
def get_slave_conn(self) -> StrictRedis:
r"""
获取slave连接
:return:
"""
raise RuntimeError("Not implemented")
class RedisServiceMasterAndSlave(BaseRedisService):
r"""
master redis和slave redis分别设置的redis帮助服务
"""
master_redis_config_dict_list: List[Dict] = None
slave_redis_config_dict_list: List[Dict] = None
master_redis_conn_list: List[StrictRedis] = None
slave_redis_conn_list: List[StrictRedis] = None
def __init__(self, *args, **kwargs):
r"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key master_redis_config_dict_list master redis连接的dict列表
:key slave_redis_config_dict_list slave redis连接的dict列表
字典的key和value组成参考 redis.Redis()的init函数,大致内容有
host='localhost', port=6379,
db=0, password=None, socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None, socket_keepalive_options=None,
connection_pool=None, unix_socket_path=None,
encoding='utf-8', encoding_errors='strict',
charset=None, errors=None,
decode_responses=False, retry_on_timeout=False,
ssl=False, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs='required', ssl_ca_certs=None,
ssl_check_hostname=False,
max_connections=None, single_connection_client=False,
health_check_interval=0, client_name=None, username=None
"""
super().__init__(*args, **kwargs)
self.master_redis_config_dict_list = kwargs.get("master_redis_config_dict_list")
self.slave_redis_config_dict_list = kwargs.get("slave_redis_config_dict_list")
self.need_decode_responses = False if self.master_redis_config_dict_list[0].get("decode_responses") else True
encoding_used_for_decode_response = self.master_redis_config_dict_list[0].get("encoding")
self.encoding_used_for_decode_response = encoding_used_for_decode_response if encoding_used_for_decode_response else 'utf-8'
self._init_redis_conn()
def get_master_conn(self):
return self.master_redis_conn_list[random.choice(range(len(self.master_redis_conn_list)))]
def get_slave_conn(self):
return self.slave_redis_conn_list[random.choice(range(len(self.slave_redis_conn_list)))]
def on_operation_failed(self):
self._init_redis_conn()
def _init_redis_conn(self):
self.master_redis_conn_list = [redis.StrictRedis(**tmp_dict) for tmp_dict in self.master_redis_config_dict_list]
self.slave_redis_conn_list = [redis.StrictRedis(**tmp_dict) for tmp_dict in self.slave_redis_config_dict_list]
class RedisServiceSentinel(BaseRedisService):
r"""
哨兵版redis集群的redis帮助服务
"""
redis_sentinel_config_dict: dict = None
sentinel: Sentinel = None
service_name: str = None
def __init__(self, *args, **kwargs):
r"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key service_name: 通过哨兵查询redis server使用的服务名
:key redis_sentinel_config_dict redis哨兵连接的dict配置信息
字典的key和value组成参考 redis.Sentinel 的init函数,大致内容有
sentinels 一组节点,每个节点使用 (hostname, port) 表示
min_other_sentinels 为一个哨兵定义一个peers的最小节点数,当查询一个哨兵时,如果没有达到这个阈值,其响应不被认为是有效的,默认为0
sentinel_kwargs 是一个字典,用于连接到哨兵时使用, 和 redis.Redis 的init函数中需要的参数一样,
如果没有提供该值,则使用 connection_kwargs 中以 socket_ 开头的参数作为 sentinel_kwargs
connection_kwargs 中的参数用于创建 redis server时使用
基本上 sentinel_kwargs 只需要 password=None, socket_timeout=None,
connection_kwargs 中需要 db=0, password=None, socket_timeout=None,
其中 sentinel_kwargs 和 connection_kwargs 可用的参数有
db=0, password=None, socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None, socket_keepalive_options=None,
connection_pool=None, unix_socket_path=None,
encoding='utf-8', encoding_errors='strict',
charset=None, errors=None,
decode_responses=False, retry_on_timeout=False,
ssl=False, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs='required', ssl_ca_certs=None,
ssl_check_hostname=False,
max_connections=None, single_connection_client=False,
health_check_interval=0, client_name=None, username=None
"""
super().__init__(*args, **kwargs)
self.service_name = kwargs.get("service_name")
self.redis_sentinel_config_dict = kwargs.get("redis_sentinel_config_dict")
sentinels = self.redis_sentinel_config_dict.get("sentinels")
min_other_sentinels = self.redis_sentinel_config_dict.get("min_other_sentinels")
sentinel_kwargs = self.redis_sentinel_config_dict.get("sentinel_kwargs")
connection_kwargs = self.redis_sentinel_config_dict.get("connection_kwargs")
self.need_decode_responses = False if connection_kwargs and connection_kwargs.get("decode_responses") else True
encoding_used_for_decode_response = connection_kwargs.get("encoding") if connection_kwargs else None
self.encoding_used_for_decode_response = encoding_used_for_decode_response if encoding_used_for_decode_response else 'utf-8'
if connection_kwargs:
self.sentinel = Sentinel(sentinels=sentinels, min_other_sentinels=min_other_sentinels,
sentinel_kwargs=sentinel_kwargs, **connection_kwargs)
else:
self.sentinel = Sentinel(sentinels=sentinels, min_other_sentinels=min_other_sentinels,
sentinel_kwargs=sentinel_kwargs)
def get_master_conn(self):
return self.sentinel.master_for(service_name=self.service_name)
def get_slave_conn(self):
return self.sentinel.slave_for(service_name=self.service_name)
def on_operation_failed(self):
pass
class RedisServiceCluster(BaseRedisService):
r"""
集群版redis集群的redis帮助服务
"""
redis_server_config_dict_list: List[Dict] = None
redis_server_conn_list: List[StrictRedis] = None
def __init__(self, *args, **kwargs):
r"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key redis_server_config_dict_list redis连接的dict列表
字典的key和value组成参考 redis.Redis()的init函数,大致内容有
host='localhost', port=6379,
db=0, password=None, socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None, socket_keepalive_options=None,
connection_pool=None, unix_socket_path=None,
encoding='utf-8', encoding_errors='strict',
charset=None, errors=None,
decode_responses=False, retry_on_timeout=False,
ssl=False, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs='required', ssl_ca_certs=None,
ssl_check_hostname=False,
max_connections=None, single_connection_client=False,
health_check_interval=0, client_name=None, username=None
"""
super().__init__(*args, **kwargs)
self.redis_server_config_dict_list = kwargs.get("redis_server_config_dict_list")
self.need_decode_responses = False if self.redis_server_config_dict_list[0].get("decode_responses") else True
encoding_used_for_decode_response = self.redis_server_config_dict_list[0].get("encoding")
self.encoding_used_for_decode_response = encoding_used_for_decode_response if encoding_used_for_decode_response else 'utf-8'
self._init_redis_conn()
def get_master_conn(self):
return self.redis_server_conn_list[random.choice(range(len(self.redis_server_conn_list)))]
def get_slave_conn(self):
return self.redis_server_conn_list[random.choice(range(len(self.redis_server_conn_list)))]
def on_operation_failed(self):
self._init_redis_conn()
def _init_redis_conn(self):
self.redis_server_conn_list = [redis.StrictRedis(**tmp_dict) for tmp_dict in self.redis_server_config_dict_list]
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import os
import shutil
import subprocess
import tempfile
from typing import Tuple
from ..globalutility import Utility
class GenCertResult:
root_key_content: str = None
root_key_filepath: str = None
root_cert_content: str = None
root_cert_filepath: str = None
ca_key_content: str = None
ca_key_filepath: str = None
ca_cert_content: str = None
ca_cert_filepath: str = None
cert_chain_content: str = None
def __init__(self, **kwargs):
r"""
:param kwargs:
:key root_key_content:
:key root_key_filepath:
:key root_cert_content:
:key root_cert_filepath:
:key ca_key_content:
:key ca_key_filepath:
:key ca_cert_content:
:key ca_cert_filepath:
:key cert_chain_content:
"""
self.root_key_content = kwargs.get("root_key_content")
self.root_key_filepath = kwargs.get("root_key_filepath")
self.root_cert_content = kwargs.get("root_cert_content")
self.root_cert_filepath = kwargs.get("root_cert_filepath")
self.ca_key_content = kwargs.get("ca_key_content")
self.ca_key_filepath = kwargs.get("ca_key_filepath")
self.ca_cert_content = kwargs.get("ca_cert_content")
self.ca_cert_filepath = kwargs.get("ca_cert_filepath")
self.cert_chain_content = kwargs.get("cert_chain_content")
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
class CertUtils:
r"""
证书相关的Utils类
"""
def gen_root_key_and_cert_and_cluster_key_and_cert(
self,
keysize: int = 4096, content_format: str = "pem",
**kwargs
) -> GenCertResult:
r"""
一次性生成根证书key和cert文件内容和生成中间证书key和cert文件内容
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:param kwargs:
:key rootca_org: 默认值 Istio
:key rootca_cn: 默认值 Root CA
:key rootca_days: 默认值 36500
:key intermediate_cluster_name: 必须提供,
:key dns_names: 默认值 [istiod.istio-system.svc]
:key ip_addresses: 默认值 []
:key intermediate_org: 默认值 Istio
:key intermediate_cn: 默认值 Intermediate CA
:key intermediate_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
dest_dir = kwargs.get("dest_dir") or None
if not dest_dir:
dest_dir = tempfile.mkdtemp()
del_dir = kwargs.get("del_dir")
if del_dir is None:
del_dir = True
kwargs["dest_dir"] = dest_dir
kwargs["del_dir"] = False
# 首先生成 root-key.pem 和 root-cert.pem
result1 = self.gen_root_key_and_cert(keysize=keysize, content_format=content_format, **kwargs)
# 然后生成 ca-key.pem 和 ca-cert.pem
result2 = self.gen_cluster_key_and_cert(
keysize=keysize, content_format=content_format,
root_key=result1.root_key_filepath, root_cert=result1.root_cert_filepath,
**kwargs
)
if del_dir:
shutil.rmtree(dest_dir, ignore_errors=True)
return result2
def gen_root_key_and_cert(self, keysize: int = 4096, content_format: str = "pem", **kwargs) -> GenCertResult:
r"""
一次性生成根证书key和cert文件内容
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:param kwargs:
:key rootca_org: 默认值 Istio
:key rootca_cn: 默认值 Root CA
:key rootca_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
dest_dir = kwargs.get("dest_dir") or None
if not dest_dir:
dest_dir = tempfile.mkdtemp()
del_dir = kwargs.get("del_dir")
if del_dir is None:
del_dir = True
kwargs["dest_dir"] = dest_dir
kwargs["del_dir"] = False
# 首先生成 root-key.pem
root_key_content, root_key_filepath = self.gen_key(
keysize=keysize, content_format=content_format,
dest_dir=dest_dir, del_dir=False,
key_file_name="root-key",
)
# 然后生成 root-cert.pem
root_cert_content, root_cert_filepath = self.gen_root_cert(
root_key=root_key_filepath,
keysize=keysize, content_format=content_format,
**kwargs
)
if del_dir:
shutil.rmtree(dest_dir, ignore_errors=True)
return GenCertResult(
root_key_content=root_key_content,
root_key_filepath=root_key_filepath,
root_cert_content=root_cert_content,
root_cert_filepath=root_cert_filepath,
)
def gen_cluster_key_and_cert(
self,
root_key: str, root_cert: str,
keysize: int = 4096, content_format: str = "pem",
**kwargs
) -> GenCertResult:
r"""
一次性生成中间证书key和cert文件内容
:param root_key: root_key文件内容或文件路径
:param root_cert: root_cert文件内容或文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:key intermediate_cluster_name: 必须提供,
:key dns_names: 默认值 [istiod.istio-system.svc]
:key ip_addresses: 默认值 []
:key intermediate_org: 默认值 Istio
:key intermediate_cn: 默认值 Intermediate CA
:key intermediate_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
dest_dir = kwargs.get("dest_dir") or None
if not dest_dir:
dest_dir = tempfile.mkdtemp()
del_dir = kwargs.get("del_dir")
if del_dir is None:
del_dir = True
kwargs["dest_dir"] = dest_dir
kwargs["del_dir"] = False
# 首先生成 ca-key.pem
ca_key_content, ca_key_filepath = self.gen_key(
keysize=keysize, content_format=content_format,
dest_dir=dest_dir, del_dir=False,
key_file_name="ca-key",
)
# 然后生成 ca-cert.pem
ca_cert_content, ca_cert_filepath = self.gen_intermediate_cert(
root_key=root_key, root_cert=root_cert,
ca_key=ca_key_filepath,
keysize=keysize, content_format=content_format,
**kwargs
)
# 读取 root_key 内容
if os.path.exists(root_key) and os.path.isfile(root_key):
root_key_filepath = root_key
with open(root_key, 'r', encoding='utf-8') as f:
root_key_content = f.read()
else:
root_key_filepath = os.sep.join([dest_dir, "root-key.pem"])
root_key_content = root_key
# 读取 root_cert 内容
if os.path.exists(root_cert) and os.path.isfile(root_cert):
root_cert_filepath = root_cert
with open(root_cert, 'r', encoding='utf-8') as f:
root_cert_content = f.read()
else:
root_cert_filepath = os.sep.join([dest_dir, "root-cert.pem"])
root_cert_content = root_cert
if del_dir:
shutil.rmtree(dest_dir, ignore_errors=True)
# 生成 cert_chain_content
cert_chain_content = f"{ca_cert_content}{root_cert_content}"
return GenCertResult(
root_key_content=root_key_content,
root_key_filepath=root_key_filepath,
root_cert_content=root_cert_content,
root_cert_filepath=root_cert_filepath,
ca_key_content=ca_key_content,
ca_key_filepath=ca_key_filepath,
ca_cert_content=ca_cert_content,
ca_cert_filepath=ca_cert_filepath,
cert_chain_content=cert_chain_content,
)
def gen_key(self, keysize: int = 4096, content_format: str = "pem", **kwargs) -> Tuple[str, str]:
r"""
生成key文件内容,返回 文件内容,文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:param kwargs:
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
:key key_file_name: 生成的key文件名,默认为 x-key
"""
supported_format = {"pem"}
if content_format not in supported_format:
raise ValueError(f"不支持的格式,目前支持的格式:{supported_format}")
dest_dir = kwargs.get("dest_dir") or None
if not dest_dir:
dest_dir = tempfile.mkdtemp()
del_dir = kwargs.get("del_dir")
if del_dir is None:
del_dir = True
key_file_name = kwargs.get("key_file_name") or "x-key"
x_key_filepath = os.sep.join([dest_dir, f"{key_file_name}.pem"])
# 使用命令 openssl genrsa -out root-key.pem ${ROOTCA_KEYSZ}
subprocess.run(f"openssl genrsa -out {x_key_filepath} {keysize}", shell=True, check=True, timeout=10)
# 读取文件内容获取结果
with open(x_key_filepath, 'r', encoding='utf-8') as f:
result_content = f.read()
# 尝试删除临时文件夹
if del_dir:
shutil.rmtree(dest_dir, ignore_errors=True)
return result_content, x_key_filepath
def gen_root_cert(
self,
root_key: str,
keysize: int = 4096, content_format: str = "pem", **kwargs
) -> Tuple[str, str]:
r"""
生成根证书文件内容,返回 文件内容,文件路径
:param root_key: root_key文件内容或文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:key rootca_org: 默认值 Istio
:key rootca_cn: 默认值 Root CA
:key rootca_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
supported_format = {"pem"}
if content_format not in supported_format:
raise ValueError(f"不支持的格式,目前支持的格式:{supported_format}")
# 设置默认值
rootca_org = kwargs.get("rootca_org") or "Istio"
rootca_cn = kwargs.get("rootca_cn") or "Root CA"
rootca_days = kwargs.get("rootca_days") or 36500
dest_dir = kwargs.get("dest_dir") or None
if not dest_dir:
dest_dir = tempfile.mkdtemp()
del_dir = kwargs.get("del_dir")
if del_dir is None:
del_dir = True
if os.path.exists(root_key) and os.path.isfile(root_key):
root_key_filepath = root_key
else:
# 写入 root-key文件
root_key_filepath = os.sep.join([dest_dir, "root-key.pem"])
with open(root_key_filepath, 'w', encoding='utf-8') as f:
f.write(root_key)
# 生成 root-ca.conf
root_ca_conf_filepath = os.sep.join([dest_dir, "root-ca.conf"])
with open(root_ca_conf_filepath, 'w', encoding='utf-8') as f:
f.write(f"""[ req ]
encrypt_key = no
prompt = no
utf8 = yes
default_md = sha256
default_bits = {keysize}
req_extensions = req_ext
x509_extensions = req_ext
distinguished_name = req_dn
[ req_ext ]
subjectKeyIdentifier = hash
basicConstraints = critical, CA:true
keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign
[ req_dn ]
O = {rootca_org}
CN = {rootca_cn}
""")
# 生成 root-cert.csr
root_cert_csr_filepath = os.sep.join([dest_dir, "root-cert.csr"])
# 使用命令 openssl req -new -key root-key.pem -config root-ca.conf -out root-cert.csr
subprocess.run(
f"openssl req -new -key {root_key_filepath} -config {root_ca_conf_filepath} -out {root_cert_csr_filepath}",
shell=True, check=True, timeout=10)
# 生成公钥 root-cert.pem
root_cert_filepath = os.sep.join([dest_dir, "root-cert.pem"])
# 使用命令 openssl x509 -req -days ${ROOTCA_DAYS} -signkey root-key.pem -extensions req_ext -extfile root-ca.conf -in root-cert.csr -out root-cert.pem
subprocess.run(
f"openssl x509 -req -days {rootca_days} -signkey {root_key_filepath} -extensions req_ext -extfile {root_ca_conf_filepath} -in {root_cert_csr_filepath} -out {root_cert_filepath}",
shell=True, check=True, timeout=10)
# 读取文件内容获取结果
with open(root_cert_filepath, 'r', encoding='utf-8') as f:
result_content = f.read()
# 尝试删除临时文件夹
if del_dir:
shutil.rmtree(dest_dir, ignore_errors=True)
return result_content, root_cert_filepath
def gen_intermediate_cert(
self,
root_key: str, root_cert: str, ca_key: str,
keysize: int = 4096, content_format: str = "pem", **kwargs) -> Tuple[str, str]:
r"""
生成中间证书文件内容,返回 文件内容,文件路径
:param root_key: root_key文件内容或文件路径
:param root_cert: root_cert文件内容或文件路径
:param ca_key: ca_key文件内容或文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:key dns_names: 默认值 [istiod.istio-system.svc]
:key ip_addresses: 默认值 []
:key intermediate_org: 默认值 Istio
:key intermediate_cn: 默认值 Intermediate CA
:key intermediate_cluster_name: 没有默认值
:key intermediate_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
supported_format = {"pem"}
if content_format not in supported_format:
raise ValueError(f"不支持的格式,目前支持的格式:{supported_format}")
# 设置默认值
dns_names = kwargs.get("dns_names") or ["istiod.istio-system.svc"]
ip_addresses = kwargs.get("ip_addresses") or None
intermediate_org = kwargs.get("intermediate_org") or "Istio"
intermediate_cn = kwargs.get("intermediate_cn") or "Intermediate CA"
intermediate_days = kwargs.get("intermediate_days") or 36500
intermediate_cluster_name = kwargs.get("intermediate_cluster_name") or ""
if not intermediate_cluster_name:
raise ValueError(f"未提供intermediate_cluster_name")
dest_dir = kwargs.get("dest_dir") or None
if not dest_dir:
dest_dir = tempfile.mkdtemp()
del_dir = kwargs.get("del_dir")
if del_dir is None:
del_dir = True
# 写入 root-key文件
if os.path.exists(root_key) and os.path.isfile(root_key):
root_key_filepath = root_key
else:
# 写入 root-key文件
root_key_filepath = os.sep.join([dest_dir, "root-key.pem"])
with open(root_key_filepath, 'w', encoding='utf-8') as f:
f.write(root_key)
if os.path.exists(root_cert) and os.path.isfile(root_cert):
root_cert_filepath = root_cert
else:
# 写入 root-cert文件
root_cert_filepath = os.sep.join([dest_dir, "root-cert.pem"])
with open(root_cert_filepath, 'w', encoding='utf-8') as f:
f.write(root_cert)
if os.path.exists(ca_key) and os.path.isfile(ca_key):
ca_key_filepath = ca_key
else:
# 写入 ca_key 文件
ca_key_filepath = os.sep.join([dest_dir, "ca-key.pem"])
with open(ca_key_filepath, 'w', encoding='utf-8') as f:
f.write(ca_key)
# 生成 intermediate.conf
intermediate_conf_filepath = os.sep.join([dest_dir, "intermediate.conf"])
# 生成 san 内容
dns_and_ip_list = list()
if dns_names:
for index, dns_name in enumerate(dns_names):
dns_and_ip_list.append(f"DNS.{index} = {dns_name}")
if ip_addresses:
for index, ip_address in enumerate(ip_addresses):
dns_and_ip_list.append(f"IP.{index} = {ip_address}")
dns_and_ip_joined_str = "\n".join(dns_and_ip_list)
with open(intermediate_conf_filepath, 'w', encoding='utf-8') as f:
f.write(f"""[ req ]
encrypt_key = no
prompt = no
utf8 = yes
default_md = sha256
default_bits = {keysize}
req_extensions = req_ext
x509_extensions = req_ext
distinguished_name = req_dn
[ req_ext ]
subjectKeyIdentifier = hash
basicConstraints = critical, CA:true, pathlen:0
keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign
subjectAltName=@san
[ san ]
{dns_and_ip_joined_str}
[ req_dn ]
O = {intermediate_org}
CN = {intermediate_cn}
L = {intermediate_cluster_name}
""")
# 生成 ca-cert.csr
ca_cert_csr_filepath = os.sep.join([dest_dir, "ca-cert.csr"])
# 使用命令 openssl req -new -key ca-key.pem -config intermediate.conf -out ca-cert.csr
subprocess.run(
f"openssl req -new -key {ca_key_filepath} -config {intermediate_conf_filepath} -out {ca_cert_csr_filepath}",
shell=True, check=True, timeout=10)
# 生成公钥 ca-cert.pem
ca_cert_filepath = os.sep.join([dest_dir, "ca-cert.pem"])
# 使用命令 openssl x509 -req -days ${INTERMEDIATE_DAYS} -CA ../root-cert.pem -CAkey ../root-key.pem -CAcreateserial -extensions req_ext -extfile intermediate.conf -in ca-cert.csr -out ca-cert.pem
subprocess.run(
f"openssl x509 -req -days {intermediate_days} -CA {root_cert_filepath} -CAkey {root_key_filepath} -CAcreateserial -extensions req_ext -extfile {intermediate_conf_filepath} -in {ca_cert_csr_filepath} -out {ca_cert_filepath}",
shell=True, check=True, timeout=10)
# 读取文件内容获取结果
with open(ca_cert_filepath, 'r', encoding='utf-8') as f:
result_content = f.read()
# 尝试删除临时文件夹
if del_dir:
shutil.rmtree(dest_dir, ignore_errors=True)
return result_content, ca_cert_filepath
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import json
import logging
import os
import traceback
from typing import List, Dict, Union
import yaml
from .base_const import ConstBaseApp
class CommonAppConfig:
r"""
应用的通用配置类,通过 CommonAppConfig() 获取
"""
# 通用配置
is_debug: bool = False
log_level_dict = {"debug": logging.DEBUG, "info": logging.INFO, "warn": logging.WARN,
"warning": logging.WARNING,
"error": logging.ERROR, "fatal": logging.FATAL, "critical": logging.CRITICAL}
# 初始化字典中log_level使用字符串,初始化后转换为对应的log level int值
log_level: int = None
runtime_env: str = None
_common_logger: logging.Logger = None
r"""通用的debug使用的logger"""
# 运行时内存缓存
runtime_cache_dict: dict = None
def __new__(cls, *args, **kwargs):
if not hasattr(cls, 'inst'): # 单例
self = super().__new__(cls, *args, **kwargs)
# 通过环境变量初始化,应用要连接的数据库、redis等其他配置
self.init_app_config()
cls.inst = self
return getattr(cls, 'inst')
def init_app_config(self):
is_debug_str = os.environ.get("IS_DEBUG") or "false"
self.is_debug = is_debug_str.lower() == "true"
if self.is_debug:
self.log_level = logging.DEBUG
else:
log_level_str = os.environ.get("LOG_LEVEL") or "info"
log_level_int = self.log_level_dict.get(log_level_str.lower())
if log_level_int is None:
log_level_int = logging.INFO
self.log_level = log_level_int
self.runtime_env = os.environ.get("RUNTIME_ENV") or "NOTSET"
def refresh_app_config(self, conf_filepath: str = ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH):
if not conf_filepath:
conf_filepath = ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH
try:
app_config_dict = self.load_file_as_dict(conf_filepath)
except BaseException as e:
if conf_filepath != ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH:
self.common_logger.warning(f"读取配置文件{conf_filepath}时发生异常:{e},不刷新CommonAppConfig")
self.common_logger.warning(traceback.format_exc())
return
# 加载默认值
self.init_app_config()
for conf_key in {"runtime_env"}:
conf_value = app_config_dict.get(conf_key)
if conf_value is not None:
setattr(self, conf_key, conf_value)
is_debug = app_config_dict.get("is_debug")
if is_debug is None:
# 如果没有配置属性,则什么也不做
pass
else:
if isinstance(is_debug, bool):
self.is_debug = is_debug
else:
is_debug_str = str(is_debug)
self.is_debug = is_debug_str.lower() == "true"
if self.is_debug:
self.log_level = logging.DEBUG
else:
log_level = app_config_dict.get("log_level")
if log_level is None:
# 如果没有配置属性,则什么也不做
pass
else:
if isinstance(log_level, int):
self.log_level = log_level
else:
log_level_str = str(log_level)
log_level_int = self.log_level_dict.get(log_level_str.lower())
if log_level_int is None:
log_level_int = logging.INFO
self.log_level = log_level_int
return
def load_file_as_dict(self, filepath: str) -> dict:
r"""
判断文件后缀名, 如果是 .yaml 或 .yml 则按 yaml 方式读取,
文件内容不合规,则抛出异常
"""
with open(filepath, 'r', encoding="utf-8") as dict_file:
conf_content = dict_file.read()
# 无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
dict_content = self.yaml_load(conf_content)
if isinstance(dict_content, str):
raise ValueError(f"{filepath}的内容不是有效的json或yaml字符串")
return dict_content
def yaml_load(self, yaml_str: str) -> Union[dict, str]:
r"""
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
"""
dict_list = self.yaml_load_list(cfg_content=yaml_str)
if dict_list:
return dict_list[0]
else:
return dict()
def yaml_load_list(self, path: str = None, cfg_content: str = None) -> List[Union[Dict, str]]:
"""
将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据,每个dict放到list中返回,
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
:param path: 文件路径
:param cfg_content:
:return: list -> []
"""
if path:
cfg = self.read_file_content(file_path=path)
else:
cfg = cfg_content or ""
yaml_generator = yaml.safe_load_all(cfg) # 将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据
if yaml_generator is None:
return list()
# 转成dict并保存到list中
yaml_list = list()
for one_yaml_generator in yaml_generator:
if one_yaml_generator is None:
continue
yaml_list.append(json.loads(json.dumps(one_yaml_generator)))
return yaml_list
def read_file_content(self, file_path) -> str:
"""
读取文件内容
:param file_path: 文件路径
:return: str -> file-content
"""
with open(file_path, 'r') as f: # 打开一个文件,必须是'rb'模式打开
return f.read()
@property
def common_logger(self):
r"""
通用的debug使用的logger
"""
if self._common_logger is None:
logger = logging.getLogger("COMMON_LOGGER")
logger.setLevel(self.log_level)
if not logger.handlers:
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(name)s - %(pathname)s - func:%(funcName)s - lineno:%(lineno)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
ch.setFormatter(formatter)
logger.addHandler(ch)
self._common_logger = logger
return self._common_logger
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import datetime
import logging
import random
import string
import threading
import traceback
from typing import Union, Optional
from flask import Request, make_response, Response, request, g
from pymysql import Connection
from .app_exception import AppException, AppRuntimeException
from .app_response import AppResponse
from .base_const import ConstResponseCode, ConstBaseApp
from .base_db_service import DbBehaviourAfterExec, BaseDbService
from .common_app_config import CommonAppConfig
from .logined_user import LoginedUserInfo
from .utils_http.http_cookie import HttpCookie
from .utils_http.http_utils import HttpUtils
from .utils_k8s.k8s_api_object_utils import OperationResultException
from .utils_tg_configcenter.configcenter_utils import ConfigCenterUtils
from .utils_tg_iam.iam_utils import IamUtils
from .utils_tg_sso.sso_utils import SsoUtils
from .utils_v2 import UtilityV2
from ..my_stringutils import MyStringUtils
class CommonBaseWebService:
"""
基础微服务-基类,
继承类需要按照实际需要覆盖基类的 app_config() , get_app_product_code() , get_iam_domain() , get_config_center_domain() , app_utils() 方法,
还有 gen_app_db_conn() 方法, refresh_app_config() 等方法,
尤其是 refresh_app_config() 方法,如果需要动态更新应用logLevel和自己的应用配置,需要覆盖该方法
"""
need_login: bool = True
r"""
应用是否需要登录,默认为True,继承类自行覆盖
"""
_conn: Connection = None
r"""
应用使用的conn,因为conn使用完需要关闭,发生异常时可能无法关闭,所以在基类做统一的关闭处理,
如果具体的服务需要使用数据库连接,可以初始化这个属性,然后关闭操作可以交给基类
需要继承类实现 gen_app_db_conn 方法以初始化该属性, 示例:
def gen_app_db_conn(self) -> Connection:
raise AppRuntimeException(
message=f"TODO:gen_app_db_conn is To Implement",
detail=f"TODO:gen_app_db_conn is To Implement"
)
"""
no_commit_close_behaviour = DbBehaviourAfterExec.gen_no_commit_close_behaviour()
commit_and_close_behaviour = DbBehaviourAfterExec()
commit_and_no_close_behaviour = DbBehaviourAfterExec(
commit_after_exec=True,
close_conn_after_exec=False,
close_conn_when_exception=True,
rollback_when_exception=True,
commit_when_exception=False
)
_app_product_code: str = None
r"""
iam鉴权时使用的 product_code, 需要继承类实现 get_app_product_code 方法以初始化该属性, 示例:
def get_app_product_code(self) -> str:
raise AppRuntimeException(
message=f"TODO:get_app_product_code is To Implement",
detail=f"TODO:get_app_product_code is To Implement"
)
"""
# 应用使用的logger
logger: logging.Logger = None
_app_config = None
r"""
app_config , 需要继承类自行实现该属性,示例:
@property
def app_config(self) -> MyAppConfig:
if self._app_config is None:
self._app_config = MyAppConfig.load_app_config()
return self._app_config
@classmethod
def load_app_config(cls, conf_filepath: str = None):
if not conf_filepath:
env_name = ConstGen.ENV_APP_CONFIG_FILEPATH
conf_filepath = os.environ.get(env_name)
if not conf_filepath:
conf_filepath = ConstGen.DEFAULT_APP_CONFIGFILE_PATH
app_config_dict = UtilityV2().load_file_as_dict(conf_filepath)
return MyAppConfig(**app_config_dict)
"""
common_app_config: CommonAppConfig = None
r"""
应用的通用配置类
"""
is_debug: bool = False
# 应用使用的utils实例,减少utils创建次数
_app_utils = None
r"""
应用使用的utils实例,减少utils创建次数,默认为 UtilityV2 类型,
如果应用有自己的继承自 UtilityV2 的util类,需要自行覆盖该属性,示例:
@property
def app_utils(self)->AppUtils:
if self._app_utils is None:
self._app_utils=AppUtils(logger=self.logger)
return self._app_utils
"""
http_utils: HttpUtils = None
r"""
通用的http便捷类实例,减少utils创建次数
"""
request: Request = None
r"""
应用处理的request
"""
request_json: dict = None
r"""
API接收的json数据
"""
request_values: dict = None
r"""
query参数和form-data或x-www-form-urlencoded传入的参数综合,即request.args和request.form的综合,key相同时query参数优先,先出现的key优先
"""
params_json: dict = None
r"""
API接收的json数据,根据query参数或post body决定,两者都提供时, POST body 中的 json内容优先,不进行合并
"""
login_user: Optional[LoginedUserInfo] = None
r"""
访问服务的用户身份,通过 g.user_info 获取
"""
resp_cookie: HttpCookie = None
r"""
用于设置 response Cookie , API 需要设置 Cookie时,设置该属性
"""
is_login: bool = False
r"""
根据login_user是否存在,分析出的是否登录属性
"""
# 天宫个性化便捷类
_iam_domain: str = None
r"""
生成iam_utils时使用的iam_domain属性, 需要继承类实现 get_iam_domain 方法以初始化该属性, 示例:
def get_iam_domain(self) -> str:
raise AppRuntimeException(
message=f"TODO:get_iam_domain is To Implement",
detail=f"TODO:get_iam_domain is To Implement"
)
"""
_iam_utils: IamUtils = None
_config_center_domain: str = None
r"""
生成configcenter_utils时使用的config_center_domain属性, 需要继承类实现 get_config_center_domain 方法以初始化该属性, 示例:
def get_config_center_domain(self) -> str:
raise AppRuntimeException(
message=f"TODO:get_config_center_domain is To Implement",
detail=f"TODO:get_config_center_domain is To Implement"
)
"""
_configcenter_utils: ConfigCenterUtils = None
def __init__(self, a_request=None):
r"""
初始化,并获取登陆用户信息以及是否登陆状态
:param a_request:
"""
# 初始化应用的config
self.common_app_config = CommonAppConfig()
self.refresh_app_config()
a_request = a_request or request
# 初始化应用使用的logger
random_str = ''.join([random.choice(string.digits + string.ascii_letters) for _ in range(16)])
self.logger = UtilityV2.gen_logger(
logger_name=threading.currentThread().getName() + "-" + random_str,
log_level=CommonAppConfig().log_level,
)
# 根据应用的config判断当前是否是debug模式
self.is_debug = self.common_app_config.is_debug
# 初始化通用的http便捷类
self.http_utils: HttpUtils = HttpUtils(incoming_request=a_request, logger=self.logger)
# 初始化应用处理的request
self.request = a_request
# 初始化 self.request_values
self.request_values = self.request.values.to_dict()
# 初始化 self.request_json
# 因为 self.request.json 在请求中的 Content - Type为application / json时才不为空,
# 而且此时body必须有内容,如果没有内容,调用 self.request.json 会报异常, 所以封装一个不会抛出异常的方法
self.request_json = self.gen_request_json_content()
# 初始化 params_json
if self.request_json != {}:
self.params_json = self.request_json
else:
params_in_query = self.request.args.get('params')
if params_in_query:
self.params_json = self.app_utils.jsonstr2dict(params_in_query)
if self.params_json is None:
self.params_json = dict()
# 初始化应用的访问者身份
if hasattr(g, "user_info") and isinstance(g.user_info, dict):
self.login_user: LoginedUserInfo = SsoUtils(
sso_api="dontcare", logger=self.logger
).gen_user_info_from_sso_api_response(
action="从用户信息字典中获取用户信息",
user_info_dict=g.user_info,
access_token=request.cookies.get("accessToken"),
)
else:
self.login_user: Optional[LoginedUserInfo] = None
self.is_login = self.login_user is not None and self.login_user.is_login()
def do_service(self):
r"""
通用的服务执行模版,记录了服务访问时长以及捕获 SafeException 时返回 code, message json
请覆盖 my_do_service() 实现你自己的逻辑
:return:
"""
start_time = datetime.datetime.now()
try:
# 记录开始调用日志
self.log_start_call_info()
# 获取服务返回结果
ret_obj = self.gen_ret_obj()
# 计算服务时长,
end_time = datetime.datetime.now()
consume_ms = (end_time - start_time).total_seconds() * 1000
# 记录访问结束日志
self.log_end_call_info(consume_ms=consume_ms, ret_obj=ret_obj)
# 尝试关闭服务连接
self.try_commit_and_close_conn()
# 返回响应信息
resp = self.handle_ret_obj(ret_obj)
if self.resp_cookie and isinstance(resp, Response):
# 设置cookie
self.resp_cookie.do_set_cookie(resp)
if isinstance(resp, Response):
resp.headers["Connection"] = "close"
return resp
except AppException as safe_exception:
self.try_commit_and_close_conn()
return self.handle_app_exception(safe_exception, start_time)
except BaseException as e:
exception_tracback = traceback.format_exc()
self.try_commit_and_close_conn()
return self.handle_common_exception(e, exception_tracback, start_time)
def log_start_call_info(self):
r"""
记录开始调用日志
"""
user_name = self.login_user.user_name if self.login_user else None
user_id = self.login_user.user_id if self.login_user else None
self.logger.info(self.app_utils.join_str_with_none(
f"{user_name}({user_id}) 服务调用:",
self.__class__.__name__,
" 开始处理时间:",
self.app_utils.now_daytime(),
" values=", self.app_utils.dict2jsonstr(self.request_values),
" json=", self.app_utils.dict2jsonstr(self.request_json),
f" 访问路径:{self.request.method} {self.request.path}",
))
def log_end_call_info(self, consume_ms: float, e=None, exception_tracback=None, ret_obj=None):
r"""
记录结束调用日志
"""
user_name = self.login_user.user_name if self.login_user else None
user_id = self.login_user.user_id if self.login_user else None
if e is None:
if isinstance(ret_obj, (AppResponse, str)):
ret_str = str(ret_obj)
elif isinstance(ret_obj, dict):
ret_str = self.app_utils.dict2jsonstr(ret_obj)
elif isinstance(ret_obj, list):
ret_str = self.app_utils.list2jsonstr(ret_obj)
else:
ret_str = ret_obj
self.logger.info(self.app_utils.join_str_with_none(
f"{user_name}({user_id}) 服务调用:",
self.__class__.__name__, f" 正常结束,耗时 {consume_ms} 豪秒,返回内容:{ret_str}"
))
else:
self.logger.error(
self.app_utils.join_str_with_none(
f"{user_name}({user_id}) 服务调用:",
self.__class__.__name__, f" 异常结束,耗时 {consume_ms} 豪秒"
)
)
self.logger.error(self.join_str(self.__class__.__name__, f'服务发生异常:{e.__class__.__name__}:{e}'))
self.logger.error(exception_tracback)
def gen_ret_obj(self):
r"""
获取服务返回对象,
统一业务逻辑,减少业务代码
"""
# 统一业务逻辑,减少业务代码
if self.need_login and not self.is_login:
ret_obj = self.action_when_not_login()
else:
# 执行业务逻辑
ret_obj = self.my_do_service()
return ret_obj
def handle_ret_obj(self, ret_obj):
r"""
处理 ret_obj
"""
if isinstance(ret_obj, AppResponse):
resp = make_response(str(ret_obj), int(ret_obj.status_code))
elif isinstance(ret_obj, str):
resp = make_response(ret_obj)
elif isinstance(ret_obj, dict):
resp = make_response(self.app_utils.dict2jsonstr(ret_obj))
elif isinstance(ret_obj, list):
resp = make_response(self.app_utils.list2jsonstr(ret_obj))
else:
resp = ret_obj
return resp
def my_do_service(self):
r"""
通用的 do_service(),调用该函数,请覆盖该函数实现你自己的逻辑
:return:
"""
raise AppRuntimeException(
message=f"TODO:my_do_service is To Implement",
detail=f"TODO:my_do_service is To Implement"
)
def try_commit_and_close_conn(self):
r"""
尝试关闭conn
:return:
"""
try:
BaseDbService(logger=self.logger).commit(conn=self._conn, close_conn_after_exec=True)
except BaseException:
pass
def gen_ok_resp(self, data=None):
r"""
快速生成OK的返回信息
:param data:
:return:
"""
return AppResponse(code=ConstResponseCode.CODE_OK, data=data)
def gen_sys_error_resp(self, data=None):
r"""
快速生成系统错误的返回信息
:param data:
:return:
"""
return AppResponse(code=ConstResponseCode.CODE_SYS_ERROR, data=data)
def gen_not_login_err(self) -> AppResponse:
r"""
快速生成未登录错误
:return:
"""
return AppResponse(code=ConstResponseCode.CODE_AUTH_FAILURE)
def raise_not_login_err(self):
r"""
快速抛出未登录异常
"""
raise AppException(code=ConstResponseCode.CODE_AUTH_FAILURE)
def gen_no_auth_err(self, message: str = None) -> AppResponse:
r"""
快速生成缺少权限错误
:param message:
:return:
"""
return AppResponse(code=ConstResponseCode.CODE_UNAUTHORIZE_OPERATION, message=message)
def raise_no_auth_err(self, message: str = None):
r"""
快速生成缺少权限错误
:param message:
:return:
"""
raise AppException(code=ConstResponseCode.CODE_UNAUTHORIZE_OPERATION, message=message)
def gen_missing_param_err(self, message: str = None):
r"""
:param message:
:return:
"""
return AppResponse(code=ConstResponseCode.CODE_MISSING_PARAMETER, message=message)
def gen_status_err(self, message: str = None):
r"""
:param message:
:return:
"""
return AppResponse(code=ConstResponseCode.CODE_STATUS_ERROR, message=message)
def raise_missing_param_err(self, message: str = None):
r"""
:param message:
:return:
"""
raise AppException(code=ConstResponseCode.CODE_MISSING_PARAMETER, message=message)
def raise_status_err(self, message: str = None):
r"""
:param message:
:return:
"""
raise AppException(code=ConstResponseCode.CODE_STATUS_ERROR, message=message)
def join_str(self, *to_join_list: str, separator_str: str = '', wrapper_str: str = None,
ignore_none: bool = False) -> str:
r"""
连接 list,当值为 None 时当做 null字符串添加,但是不追加wrapper_str
:param to_join_list
:param separator_str
:param wrapper_str
:param ignore_none
:return
"""
return self.app_utils.list_join_to_str(
to_join_list,
separator_str=separator_str, wrapper_str=wrapper_str, ignore_none=ignore_none,
)
def resp_for_safe_exeception(self, safe_exception: AppException):
r"""
处理 AppException 时, 根据 AppException 返回的response
"""
return make_response(self.app_utils.dict2jsonstr(safe_exception.gen_err()), int(safe_exception.status_code))
def handle_app_exception(self, safe_exception: AppException, start_time):
r"""
do_service函数中处理应用异常
:param start_time
:param safe_exception:
:return:
"""
# 计算服务时长,
end_time = datetime.datetime.now()
consume_ms = (end_time - start_time).total_seconds() * 1000
ret_obj = self.app_utils.dict2jsonstr(safe_exception.gen_err())
# 记录访问结束日志
self.log_end_call_info(consume_ms=consume_ms, ret_obj=ret_obj)
# 计算服务时长,如果超时则通知管理员
# 返回响应信息(错误信息)
resp = make_response(ret_obj, int(safe_exception.status_code))
resp.headers["Connection"] = "close"
return resp
def resp_for_common_exeception(self, e):
r"""
处理 e 时, 根据 AppException 返回的response
"""
err_msg = f"{e.__class__.__name__}:{e}"
code = ConstResponseCode.CODE_SYS_ERROR
if isinstance(e, AppRuntimeException):
self.logger.error(e)
err_msg = e.message
code = e.code or ConstResponseCode.CODE_SYS_ERROR
if isinstance(e, OperationResultException):
self.logger.error(e)
err_msg = e.failed_reason
code = e.code or ConstResponseCode.CODE_SYS_ERROR
response = AppResponse(
code=code,
message=err_msg,
)
return make_response(str(response), int(response.status_code))
def handle_common_exception(self, e, exception_tracback, start_time):
r"""
do_service函数中处理通用的异常
:param e:
:param exception_tracback:
:param start_time:
:return:
"""
# 计算服务时长,
end_time = datetime.datetime.now()
consume_ms = (end_time - start_time).total_seconds() * 1000
# 记录访问结束日志
self.log_end_call_info(consume_ms=consume_ms, e=e, exception_tracback=exception_tracback)
# 通知管理员
resp = self.resp_for_common_exeception(e)
resp.headers["Connection"] = "close"
return resp
def action_when_not_login(self):
r"""
do_service时如果未登录的行为,默认为需要登录则返回未登录异常
:return:
"""
return self.gen_not_login_err()
def raise_uims_when_not_login(self):
r"""
生成带有uims返回地址的异常
:return:
"""
uims_api_for_frontpage = ""
response: AppResponse = AppResponse()
response.code = ConstResponseCode.CODE_AUTH_FAILURE
response.message = "请先登录uims并通过uims跳转到天宫后重试"
response.data = uims_api_for_frontpage
app_exception = AppException()
app_exception.my_response = response
raise app_exception
def gen_request_json_content(self):
r"""
生成 request_json的内容
封装一个不会抛出异常的方法
:return:
"""
try:
return self.app_utils.jsonstr2dict(str(self.request.data, encoding="utf-8")) or dict()
except BaseException as e:
# 打印一下错误信息
self.logger.error("获取self.request.json时发生异常,使用空的dict,异常信息如下:")
self.logger.exception(e, exc_info=True)
# 异常时返回空的dict
return dict()
def gen_user_info(self, login_user: LoginedUserInfo = None, output_type: str = "str") -> Union[str, dict]:
r"""
生成 {"user_id":"","user_name":""}格式的字典或字符串
:param login_user: 用户, 默认为 self.login_user
:param output_type: str or dict, 输出格式,默认为 str
"""
login_user = login_user or self.login_user
return self.app_utils.gen_user_info(login_user=login_user, output_type=output_type)
def gen_app_db_conn(self) -> Connection:
r"""
:return:
"""
raise AppRuntimeException(
message=f"TODO:gen_app_db_conn is To Implement",
detail=f"TODO:gen_app_db_conn is To Implement"
)
@property
def conn(self) -> Connection:
r"""
conn 属性
:return:
"""
if self._conn is None:
self._conn = self.gen_app_db_conn()
return self._conn
@conn.setter
def conn(self, conn):
r"""
conn 属性
:param conn:
:return:
"""
self._conn = conn
def get_iam_domain(self) -> str:
r"""
iam_domain 属性
"""
raise AppRuntimeException(
message=f"TODO:get_iam_domain is To Implement",
detail=f"TODO:get_iam_domain is To Implement"
)
@property
def iam_domain(self) -> str:
r"""
iam_domain 属性
:return:
"""
if not self._iam_domain:
self._iam_domain = self.get_iam_domain()
return self._iam_domain
@iam_domain.setter
def iam_domain(self, iam_domain):
r"""
iam_domain 属性
:param iam_domain:
:return:
"""
self._iam_domain = iam_domain
@property
def iam_utils(self):
r"""
iam_utils 属性
:return:
"""
if self._iam_utils is None:
self._iam_utils: IamUtils = IamUtils(
iam_api=self.iam_domain + "/iam",
access_token=self.login_user.access_token if self.login_user else MyStringUtils.EMPTY,
http_utils=self.http_utils,
app_utils=self.app_utils
)
return self._iam_utils
def get_config_center_domain(self) -> str:
r"""
config_center_domain 属性
"""
raise AppRuntimeException(
message=f"TODO:get_config_center_domain is To Implement",
detail=f"TODO:get_config_center_domain is To Implement"
)
@property
def config_center_domain(self) -> str:
r"""
config_center_domain 属性
:return:
"""
if not self._config_center_domain:
self._config_center_domain = self.get_config_center_domain()
return self._config_center_domain
@config_center_domain.setter
def config_center_domain(self, config_center_domain):
r"""
config_center_domain 属性
:param config_center_domain:
:return:
"""
self._config_center_domain = config_center_domain
@property
def configcenter_utils(self) -> ConfigCenterUtils:
r"""
configcenter_utils 属性
:return:
"""
if self._configcenter_utils is None:
self._configcenter_utils: ConfigCenterUtils = ConfigCenterUtils(
configcenter_api=self.config_center_domain + "/configcenter",
access_token=self.login_user.access_token if self.login_user else MyStringUtils.EMPTY,
http_utils=self.http_utils,
app_utils=self.app_utils
)
return self._configcenter_utils
@property
def app_utils(self) -> UtilityV2:
r"""
app_utils 属性
:return:
"""
if self._app_utils is None:
self._app_utils = UtilityV2(logger=self.logger)
return self._app_utils
@app_utils.setter
def app_utils(self, app_utils):
r"""
app_utils 属性
:param app_utils:
:return:
"""
self._app_utils = app_utils
@property
def app_config(self):
r"""
app_config 属性
:return:
"""
if self._app_config is None:
raise AppRuntimeException(
message=f"TODO:app_config is To Implement",
detail=f"TODO:app_config is To Implement"
)
return self._app_config
@app_config.setter
def app_config(self, app_config):
r"""
app_config 属性
:param app_config:
:return:
"""
self._app_config = app_config
def get_app_product_code(self) -> str:
r"""
app_product_code 属性
"""
raise AppRuntimeException(
message=f"TODO:get_app_product_code is To Implement",
detail=f"TODO:get_app_product_code is To Implement"
)
@property
def app_product_code(self) -> str:
r"""
app_product_code 属性
:return:
"""
if self._app_product_code is None:
self._app_product_code = self.get_app_product_code()
return self._app_product_code
@app_product_code.setter
def app_product_code(self, app_product_code):
r"""
app_product_code 属性
:param app_product_code:
:return:
"""
self._app_product_code = app_product_code
def refresh_app_config(self):
r"""
刷新 common_app_config 配置 和 应用配置,
默认为每次都读取 ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH 文件刷新 common_app_config ,
如果继承类需要每次都重新抓取应用配置,则需要覆盖该方法 示例:
def refresh_app_config(self):
hot_reload_from_env = os.environ.get('HOT_RELOAD_CONFIG')
hot_reload = hot_reload_from_env == "true"
if hot_reload:
# 使用应用提供的配置文件刷新 common_app_config
self.common_app_config.refresh_app_config(conf_filepath=MyAppConfig.get_app_conf_filepath())
# 使用应用提供的配置文件刷新 app_config
self.app_config.refresh_app_config(conf_filepath=MyAppConfig.get_app_conf_filepath())
或者
def refresh_app_config(self):
# 仍然使用 ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH 刷新 common_app_config
super().refresh_app_config()
hot_reload_from_env = os.environ.get('HOT_RELOAD_CONFIG')
hot_reload = hot_reload_from_env == "true"
if hot_reload:
# 使用应用提供的配置文件刷新 app_config
self.app_config.refresh_app_config(conf_filepath=MyAppConfig.get_app_conf_filepath())
"""
# 使用 ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH 刷新 common_app_config
self.common_app_config.refresh_app_config(ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH)
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from flask import make_response, Response
from .app_exception import AppRuntimeException, AppException
from .app_response import AppResponse
from .common_base_webservice import CommonBaseWebService
from .utils_k8s.k8s_api_object_utils import OperationResultException
class CommonBaseWebServiceForStream(CommonBaseWebService):
"""
基础微服务-基类(文件流类型的微服务基础),依赖一些其他应用基础类
"""
def action_when_not_login(self):
return make_response("未登录", 400)
def handle_ret_obj(self, ret_obj):
r"""
处理 ret_obj
"""
# 如果返回 AppResponse,必然是CODE不为OK的Response
if isinstance(ret_obj, AppResponse):
resp = make_response(ret_obj.message, 400)
elif isinstance(ret_obj, Response):
resp = ret_obj
else:
raise AppRuntimeException(
message="服务内部错误:处理结果类型错误",
detail=f"ret_obj类型={type(ret_obj)}, ret_obj={ret_obj}"
)
return resp
def resp_for_safe_exeception(self, safe_exception: AppException):
r"""
处理 AppException 时, 根据 AppException 返回的response
"""
return make_response(safe_exception.message, 400)
def resp_for_common_exeception(self, e):
r"""
处理 e 时, 根据 AppException 返回的response
"""
err_msg = f"{e.__class__.__name__}:{e}"
if isinstance(e, AppRuntimeException):
self.logger.error(e)
err_msg = e.message
if isinstance(e, OperationResultException):
self.logger.error(e)
err_msg = e.failed_reason
return make_response(f"服务发生异常:{err_msg}", 500)
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from .app_exception import AppRuntimeException
from ..globalutility import Utility
class LoginedUserInfo:
r"""
登陆用户的信息POJO类,不依赖任何其他应用基础类
"""
def __init__(self):
self.account_id = None
self.account_name = None
self.user_id = None
self.user_name = None
self.mobile = None
self.email = None
self.is_root = None
self.session_uuid = None
r"""
目前获取不到session_uuid
"""
self.access_token = None
r"""
继续调用其他API时使用的access_token
"""
def is_login(self):
return not (self.user_id is None or self.user_id == '')
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
@classmethod
def from_dict(cls, user_info_dict, access_token: str = None):
r"""
根据sso_api返回的用户信息dict转换成LoginedUserInfo
:param user_info_dict
:param access_token:
"""
return cls().fill_attr_by_dict(user_info_dict, access_token)
def fill_attr_by_dict(self, user_info_dict: dict, access_token: str = None):
r"""
根据sso_api返回的用户信息dict填充自身属性,返回自身
:param user_info_dict:
:param access_token:
:return:
"""
tmp_value = user_info_dict.get('accountID')
if not tmp_value:
raise AppRuntimeException(
message=f"通过字典转换成LoginedUserInfo对象失败",
detail=f"字典内容不包括accountID,user_info_dict={Utility.dict2jsonstr(user_info_dict)}"
)
self.account_id = tmp_value
tmp_value = user_info_dict.get('userID')
if not tmp_value:
raise AppRuntimeException(
message=f"通过字典转换成LoginedUserInfo对象失败",
detail=f"字典内容不包括userID,user_info_dict={Utility.dict2jsonstr(user_info_dict)}"
)
self.user_id = tmp_value
tmp_value = user_info_dict.get('userName')
if not tmp_value:
raise AppRuntimeException(
message=f"通过字典转换成LoginedUserInfo对象失败",
detail=f"字典内容不包括userName,user_info_dict={Utility.dict2jsonstr(user_info_dict)}"
)
self.user_name = tmp_value
self.account_name = user_info_dict.get('accountName')
self.mobile = user_info_dict.get('mobile')
self.email = user_info_dict.get('email')
self.is_root = user_info_dict.get('isRoot')
self.access_token = access_token
return self
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import hashlib
class Md5Utils:
r"""
md5 utils类,不依赖任何其他应用基础类
"""
def md5_hash(self, file_path, read_byte_once=1024):
r"""
获取文件的md5哈希值
"""
md5_1 = hashlib.md5() # 创建一个md5算法对象
with open(file_path, 'rb') as f: # 打开一个文件,必须是'rb'模式打开
while 1:
data = f.read(read_byte_once) # 由于是一个文件,每次只读取固定字节
if data: # 当读取内容不为空时对读取内容进行update
md5_1.update(data)
else: # 当整个文件读完之后停止update
break
ret = md5_1.hexdigest() # 获取这个文件的MD5值
return ret
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import base64
import rsa
class CannotDecryptContentException(Exception):
"""
自定义的异常类, 用于表示不能将加密后的内容进行界面的异常
"""
def __init__(self, message: str):
r"""
初始化
:param message
"""
self.message: str = message
def __str__(self):
r"""
字符串表示
Returns:
"""
return self.message
def __repr__(self):
r"""
字符串表示
Returns:
"""
return self.message
class RsaUtils:
r"""
RSA utils类,不依赖任何其他应用基础类
"""
def __init__(self, key: str = None):
r"""
初始化
:param key: pkcs1格式,即 -----BEGIN RSA PRIVATE KEY-----
"""
self.key: str = key
def encrypt(self, to_encrypt_content: str, public_key: str):
r"""
RSA加密
:param to_encrypt_content: 待加密的字符串
:param public_key: 公钥,pkcs1格式,即 -----BEGIN RSA PUBLIC KEY-----
:return: 加密后的字符串
"""
to_encrypt_content = to_encrypt_content.encode("utf-8")
rsa_public_key = rsa.PublicKey.load_pkcs1(public_key.encode("utf-8"), format='PEM')
return base64.b64encode(rsa.encrypt(to_encrypt_content, rsa_public_key)).decode("utf-8")
def gen_rsa_key(self) -> tuple:
r"""
生成公钥私钥
:return:
"""
return rsa.newkeys(512)
def decrypt(self, to_decrypt_content: str, key: str = None) -> str:
r"""
RSA解密
:param to_decrypt_content
:param key:pkcs1格式,即 -----BEGIN RSA PRIVATE KEY-----
"""
real_key = key if key else self.key
private_key = rsa.PrivateKey.load_pkcs1(real_key.encode("utf-8"), format='PEM')
try:
content = rsa.decrypt(base64.b64decode(to_decrypt_content), private_key)
return content.decode("utf-8")
except BaseException as e:
encountered_e = CannotDecryptContentException(str(e))
if encountered_e:
raise encountered_e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import logging
import smtplib
from email.header import Header
from email.message import Message
from email.mime.text import MIMEText
from typing import Union
import requests
from ..globalutility import Utility
from ..my_stringutils import MyStringUtils
class ToReceiveNoticeUser:
r"""
接收通知人员POJO类,不依赖任何其他应用基础类
"""
phone: str = None
email: str = None
def __init__(self, source: dict = None, phone: str = None, email: str = None):
r"""
根据 source(包含 phone 和 email 初始化 ToReceiveNoticeUser
:param source:
:param phone:
:param email:
"""
if source is None:
source = dict()
self.phone = phone or source.get("phone")
self.email = email or source.get("email")
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
class NoticeSendResult:
r"""
发送通知结果POJO类,不依赖任何其他应用基础类
"""
email_send_result: bool = None
sms_send_result: bool = None
def __init__(self):
r"""
初始化
"""
self.email_send_result = False
self.sms_send_result = False
def is_all_success(self):
r"""
是否全部发送成功便捷方法
"""
return self.email_send_result and self.sms_send_result
def is_any_success(self):
r"""
是否任意发送成功便捷方法
"""
return self.email_send_result or self.sms_send_result
class SendEmailConfig:
r"""
发送邮件配置类,不依赖任何其他应用基础类
"""
def __init__(self, source: dict = None, mail_host: str = None, mail_host_port: str = None,
mail_from_account: str = None,
mail_from_account_dwp: str = None):
r"""
根据 source(包含 mail_host 和 mail_from_account 和 mail_from_account_dwp key的dict) 初始化 SendEmailConfig
:param source:
:param mail_host:
:param mail_host_port:
:param mail_from_account:
:param mail_from_account_dwp:
"""
if source is None:
source = dict()
self.mail_host: str = mail_host or source.get("mail_host")
self.mail_host_port: str = mail_host_port or source.get("mail_host_port") or 25
self.mail_from_account: str = mail_from_account or source.get("mail_from_account")
self.mail_from_account_dwp: str = mail_from_account_dwp or source.get("mail_from_account_dwp")
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
class SendSmsConfig:
r"""
发送短信配置类,不依赖任何其他应用基础类
"""
def __init__(self, source: dict = None, api_server: str = None, sys_name: str = None,
sys_token: str = None):
r"""
根据 source(包含 api_server 和 sys_name 和 sys_token key的dict) 初始化 SendSmsConfig
:param source:
:param api_server:
:param sys_name:
:param sys_token:
"""
if source is None:
source = dict()
self.api_server: str = api_server or source.get("api_server")
self.sys_name: str = sys_name or source.get("sys_name")
self.sys_token: str = sys_token or source.get("sys_token")
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
class SendNoticeUtils:
r"""
发送通知 Utils 类,不依赖任何其他应用基础类
"""
send_email_config: SendEmailConfig = None
send_sms_config: SendSmsConfig = None
def __init__(self, send_email_config: SendEmailConfig = None, send_sms_config: SendSmsConfig = None):
r"""
初始化
:param send_sms_config:
:param send_email_config:
"""
self.send_email_config = send_email_config
self.send_sms_config = send_sms_config
def gen_to_receive_notice_user_list(self, user_list: list) -> list:
r"""
根据 source(包含 phone 和 email key的dict) 生成 ToReceiveNoticeUser 列表
:param user_list:
"""
if not user_list:
return list()
result_list = list()
for source in user_list:
if isinstance(source, ToReceiveNoticeUser):
tmp_source = source
else:
tmp_source = ToReceiveNoticeUser(source)
result_list.append(tmp_source)
return result_list
def do_notice(self, to_receive_notice_user_list: list, subject: str = None, notice_content: str = None,
add_subject_to_notice_content: bool = False) -> NoticeSendResult:
r"""
通知(短信通知,邮件通知等)
:param to_receive_notice_user_list:
:param subject:
:param notice_content:
:param add_subject_to_notice_content:
"""
to_receive_notice_user_list = self.gen_to_receive_notice_user_list(to_receive_notice_user_list)
email_list = [tmp_to_receive_notice_user.email for
tmp_to_receive_notice_user in to_receive_notice_user_list]
phone_list = [tmp_to_receive_notice_user.phone for
tmp_to_receive_notice_user in to_receive_notice_user_list]
real_notice_content = notice_content
if add_subject_to_notice_content:
real_notice_content = Utility.join_str(subject, '\n', notice_content)
send_success_for_send_email: bool = self.do_send_email(email_list, subject, real_notice_content)
send_success_for_send_sms: bool = self.do_send_sms(phone_list, real_notice_content)
result = NoticeSendResult()
result.sms_send_result = send_success_for_send_sms
result.email_send_result = send_success_for_send_email
return result
def do_notice_with_any_success(self, to_receive_notice_user_list: list, subject: str = None,
notice_content: str = None,
add_subject_to_notice_content: bool = False) -> bool:
r"""
通知(短信通知,邮件通知等)
:param to_receive_notice_user_list:
:param subject:
:param notice_content:
:param add_subject_to_notice_content:
"""
return self.do_notice(to_receive_notice_user_list, subject, notice_content,
add_subject_to_notice_content).is_any_success()
def do_send_email(self, to_mails: list, subject: str = None, notice_content: Union[str, Message] = None) -> bool:
r"""
发送邮件
:param to_mails:
:param subject:
:param notice_content:
"""
smtp = None
try:
if self.send_email_config is None:
raise RuntimeError('send_email_config为空')
mail_host = self.send_email_config.mail_host
mail_host_port = self.send_email_config.mail_host_port
mail_from_account = self.send_email_config.mail_from_account
mail_from_account_dwp = self.send_email_config.mail_from_account_dwp
if not mail_host:
raise RuntimeError('send_email_config.mail_host 为空')
if not mail_host_port:
raise RuntimeError('send_email_config.mail_host_port 为空')
if not mail_from_account:
raise RuntimeError('send_email_config.mail_from_account 为空')
if not mail_from_account_dwp:
raise RuntimeError('send_email_config.mail_from_account_dwp 为空')
smtp = smtplib.SMTP(host=mail_host, port=mail_host_port)
smtp.login(mail_from_account, mail_from_account_dwp)
real_notice_content = notice_content
if not isinstance(notice_content, Message):
real_notice_content = MIMEText(MyStringUtils.to_str(notice_content), "plain", "utf-8")
real_notice_content['Subject'] = Header(subject, 'utf-8')
smtp.sendmail(mail_from_account, to_mails, real_notice_content.as_string())
return True
except BaseException as e:
logging.error("发送邮件时发生异常")
logging.exception(e, exc_info=True)
return False
finally:
if smtp is not None:
smtp.quit()
def do_send_sms(self, phone_list: list, notice_content: str = None) -> bool:
r"""
发送邮件
:param phone_list:
:param notice_content:
"""
try:
if self.send_sms_config is None:
raise RuntimeError('send_sms_config为空')
api_server = self.send_sms_config.api_server
sys_name = self.send_sms_config.sys_name
sys_token = self.send_sms_config.sys_token
if not api_server:
raise RuntimeError('send_sms_config.api_server 为空')
if not sys_name:
raise RuntimeError('send_sms_config.sys_name 为空')
if not sys_token:
raise RuntimeError('send_sms_config.sys_token 为空')
schema_tuple = ("http://", "https://")
default_schema = 'http://'
if not api_server.lower().startswith(schema_tuple):
api_server = Utility.join_str(default_schema, api_server)
map_header = dict()
map_header['Content-Type'] = 'application/json'
request_dict = dict()
request_dict['sysName'] = sys_name
request_dict['token'] = sys_token
request_dict['content'] = notice_content
request_dict['phoneNoList'] = phone_list
resp = requests.post(api_server, data=Utility.dict2jsonstr(request_dict), headers=map_header, verify=False)
# 判断状态码
if resp.status_code != 200:
raise RuntimeError(Utility.join_str(f"调用sms ApiServer接口返回状态码为{resp.status_code}"))
# 判断RetCode
resp_dict = Utility.jsonstr2dict(resp.text)
if resp_dict is None:
raise RuntimeError(Utility.join_str("调用sms ApiServer接口返回内容不是jsonObject,返回内容为: ", resp.text))
ret_code = resp_dict.get('RetCode')
if str(ret_code) != '1':
raise RuntimeError(Utility.join_str("调用sms ApiServer接口返回RetCode不是1, 返回内容为: ", resp.text))
return True
except BaseException as e:
logging.error("发送短信时发生异常")
logging.exception(e, exc_info=True)
return False
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import logging
from typing import Optional, List
from .app_exception import AppException
from .base_const import ConstResponseCode
from .common_app_config import CommonAppConfig
class UtilMethods:
@classmethod
def get_intvalue_from_dict_and_check_range(
cls,
source_dict: dict,
key_name: str,
min_value: Optional[int] = None,
max_value: Optional[int] = None,
other_allowed_values: List[int] = None,
action_desc: str = None,
check_null: bool = True,
default_value: int = 0,
) -> int:
r"""
从字典中获取指定key的值,并检查其是否是int,并检查其值的范围
:param source_dict:
:param key_name:
:param min_value:可选值,提供则判断最小值,
:param max_value:可选值
:param other_allowed_values: 允许的值范围,例如 [-1], 在此范围内则不校验 最小/最大值
:param action_desc: 抛出异常时的描述
:param check_null: 默认为True,即值为None是抛出异常
:param default_value: 默认值, None表示0
"""
action_desc = action_desc or "从字典中获取字符串值"
if not key_name:
result = source_dict
else:
result = source_dict.get(key_name)
if result is None:
if check_null:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时未提供{key_name}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result})为空"
)
else:
result = default_value if default_value is not None else 0
if not isinstance(result, int):
if key_name:
raise AppException(
code=ConstResponseCode.CODE_SYS_ERROR,
message=f"{action_desc}时未提供{key_name},使用默认值 {result},但不是int类型"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result})为空,使用默认值 {result},但不是int类型"
)
elif not isinstance(result, int):
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}不是int类型"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result})不是int类型"
)
if other_allowed_values and result in other_allowed_values:
return result
if min_value is not None and result < min_value:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}的值小于{min_value}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result})小于{min_value}"
)
if max_value is not None and result > max_value:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}的值大于{max_value}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result})大于{max_value}"
)
return result
@classmethod
def get_strvalue_from_dict(
cls,
source_dict: dict,
key_name: str,
check_null_or_empty: bool = True,
default_value: str = None,
action_desc: str = None
) -> str:
r"""
从字典获取字符串,
:param source_dict:
:param key_name:
:param check_null_or_empty:是否检查值是否为空或null,默认为True
:param default_value: 默认值,默认为空字符串
:param action_desc: 抛出异常时的描述
"""
if not key_name:
result_obj = source_dict
else:
result_obj = source_dict.get(key_name)
if result_obj is not None:
result_obj = f"{result_obj}"
if check_null_or_empty and not result_obj:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时未提供{key_name}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据为空"
)
return result_obj or default_value or ""
@classmethod
def get_boolvalue_from_dict(
cls,
source_dict: dict,
key_name: str,
check_null: bool = True,
default_value: bool = False,
action_desc: str = None
) -> bool:
r"""
从字典获取bool值, 注意 default_value 是 None 时, 默认值为 None
:param source_dict:
:param key_name:
:param check_null:是否检查值是否为空或null,默认为True
:param default_value: 默认值,默认为 False
:param action_desc: 抛出异常时的描述
"""
if not key_name:
result_obj = source_dict
else:
result_obj = source_dict.get(key_name)
if result_obj is None:
if check_null:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时未提供{key_name}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result_obj})为None"
)
else:
result_obj = default_value
if not isinstance(result_obj, bool):
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}不是bool类型"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result_obj})不是bool类型"
)
return result_obj
@classmethod
def get_objvalue_from_dict(
cls,
source_dict: dict,
key_name: str,
obj_class,
action_desc: str = None,
check_null: bool = True,
**kwargs
):
r"""
从字典获取对象,
:param source_dict:
:param key_name:
:param obj_class:
:param action_desc:
:param check_null:
:param kwargs:
"""
if not key_name:
result_obj = source_dict
else:
result_obj = source_dict.get(key_name)
if isinstance(result_obj, obj_class):
return result_obj
elif isinstance(result_obj, dict):
kwargs.update(result_obj)
return obj_class().init_by_create_dict(**kwargs)
elif result_obj is None:
if check_null:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时未提供{key_name}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result_obj})为空"
)
else:
return None
else:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}为不支持的类型"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result_obj})为不支持的类型"
)
@classmethod
def get_enumvalue_from_dict(
cls,
source_dict: dict,
key_name: str,
obj_class,
action_desc: str = None,
logger: logging.Logger = None,
check_null: bool = True,
default_value=None,
):
r"""
从字典获取枚举类型对象
:param source_dict:
:param key_name:
:param obj_class
:param action_desc:
:param logger:
:param check_null:
:param default_value:
"""
logger = logger or CommonAppConfig().common_logger
if not key_name:
result_obj = source_dict
else:
result_obj = source_dict.get(key_name)
if isinstance(result_obj, obj_class):
return result_obj
elif result_obj is None:
if check_null:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时未提供{key_name}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result_obj})为None"
)
else:
return default_value
else:
try:
return obj_class(str(result_obj))
except BaseException as e:
encountered_e = e
if encountered_e:
logger.error(encountered_e)
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}不是有效的{obj_class.__name__}值类型"
).with_traceback(encountered_e.__traceback__)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({result_obj})不是有效的{obj_class.__name__}值类型"
).with_traceback(encountered_e.__traceback__)
@classmethod
def get_listvalue_from_dict(
cls,
source_dict: dict,
key_name: str,
obj_class,
check_null: bool = False,
action_desc: str = None
) -> list:
r"""
从字典获取列表,
:param source_dict:
:param key_name:
:param obj_class
:param check_null: 默认为False
:param action_desc:
"""
if not key_name:
tmp_key_value = source_dict
else:
tmp_key_value = source_dict.get(key_name)
if not tmp_key_value:
if check_null:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时未提供{key_name}"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据为空"
)
else:
return list()
if not isinstance(tmp_key_value, list):
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时提供{key_name}不是列表"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({tmp_key_value})不是列表"
)
result_obj = list()
for tmp_obj in tmp_key_value:
if isinstance(tmp_obj, obj_class):
result_obj.append(tmp_obj)
elif isinstance(tmp_obj, dict):
result_obj.append(obj_class().init_by_create_dict(**tmp_obj))
else:
if key_name:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}{key_name}中的对象是不支持的类型"
)
else:
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=f"{action_desc}时源数据({tmp_key_value})中的对象是不支持的类型"
)
return result_obj
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import base64
import hashlib
import json
import logging
import os
import pickle
import platform
import random
import re
import string
import subprocess
import tempfile
import time
import traceback
import urllib.parse
from datetime import datetime
from typing import Union, Optional, Dict, List
import yaml
from flask import Request
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from jinja2.defaults import VARIABLE_START_STRING, VARIABLE_END_STRING
from .app_exception import AppRuntimeException, AppException
from .app_response import AppResponse
from .base_const import ConstResponseCode
from .common_app_config import CommonAppConfig
from .logined_user import LoginedUserInfo
from .send_notices_utils import SendNoticeUtils
from ..globalutility import Utility
from ..my_stringutils import MyStringUtils
try:
import thread
except ImportError:
import _thread as thread
class CrnParseResult:
product_code: Optional[str] = None
region_code: Optional[str] = None
acct_id: Optional[str] = None
first_id_type: Optional[str] = None
first_id: Optional[str] = None
second_id_type: Optional[str] = None
second_id: Optional[str] = None
third_id_type: Optional[str] = None
third_id: Optional[str] = None
other_id_types: Optional[List[str]] = None
other_ids: Optional[List[str]] = None
first_crn: Optional[str] = None
second_crn: Optional[str] = None
third_crn: Optional[str] = None
other_crns: Optional[List[str]] = None
def __init__(self, **kwargs):
self.product_code = kwargs.get("product_code")
self.region_code = kwargs.get("region_code")
self.acct_id = kwargs.get("acct_id")
self.first_id_type = kwargs.get("first_id_type")
self.first_id = kwargs.get("first_id")
self.second_id_type = kwargs.get("second_id_type")
self.second_id = kwargs.get("second_id")
self.third_id_type = kwargs.get("third_id_type")
self.third_id = kwargs.get("third_id")
self.other_id_types = kwargs.get("other_id_types")
self.other_ids = kwargs.get("other_ids")
self.first_crn = kwargs.get("first_crn")
self.second_crn = kwargs.get("second_crn")
self.third_crn = kwargs.get("third_crn")
self.other_crns = kwargs.get("other_crns")
def __str__(self):
return Utility.dict2jsonstr(self.__dict__)
class UtilityBaseV2(Utility):
r"""
便捷功能类,不依赖任何类的应用的便捷功能类
"""
def __init__(self, logger: logging.Logger = None):
self.logger = logger or CommonAppConfig().common_logger
@classmethod
def gen_logger(
cls,
logger_name: str = "not_named_logger",
log_level: int = None,
):
r"""
生成logger
:param logger_name: 通常可以用 self.__class__.__name__ 来提供
:param log_level: 默认为 logging.WARNING , 默认为 CommonAppConfig().log_level
"""
logger = logging.getLogger(logger_name)
if log_level is None:
log_level = CommonAppConfig().log_level
logger.setLevel(log_level)
if not logger.handlers:
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(name)s - %(pathname)s - func:%(funcName)s - lineno:%(lineno)s - %(levelname)s - %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
@classmethod
def dict2jsonstr(cls, a_dict: dict, value_when_obj_is_none: str = 'null', sort_keys: bool = False,
remove_key_of_none_value: bool = False, attr_name_when_value_is_obj: str = '__dict__') -> str:
r"""
复用父类的 dict2jsonstr 的基础上,增加按排序key的方式生成json字符串,并且生成的字符串为最紧密的字符串
:param a_dict:
:param value_when_obj_is_none:
:param sort_keys
:param remove_key_of_none_value
:param attr_name_when_value_is_obj
:return:
"""
if a_dict is None:
return value_when_obj_is_none
elif not isinstance(a_dict, dict):
raise ValueError('参数类型不是字典')
if remove_key_of_none_value:
# 遍历dict的所有key,并递归删除value为None的key
a_dict = cls.dict_after_remove_key_of_none_value(a_dict, attr_name_when_value_is_obj)
sort_keys = True if sort_keys else False
return json.dumps(a_dict, ensure_ascii=False,
default=lambda o: cls.gen_dict_for_a_obj(o, attr_name_when_value_is_obj),
sort_keys=sort_keys, separators=(',', ':'))
@classmethod
def gen_dict_for_a_obj(cls, a_obj, attr_name):
r"""
主要用于json序列化时,value为对象时如何序列化,a_obj指定的属性或方法必须返回一个dict
:param a_obj:
:param attr_name:
:return:
"""
if not attr_name:
return getattr(a_obj, "__dict__", None)
if isinstance(attr_name, str):
if hasattr(a_obj, attr_name):
attr_obj = getattr(a_obj, attr_name, None)
if '__call__' in dir(attr_obj):
return attr_obj()
else:
return attr_obj
else:
return getattr(a_obj, "__dict__", None)
else:
if '__call__' in dir(attr_name):
return attr_name(a_obj)
else:
return getattr(a_obj, "__dict__", None)
@classmethod
def dict_after_remove_key_of_none_value(
cls,
a_dict: dict, attr_name_when_value_is_obj: str = '__dict__') -> Optional[dict]:
r"""
遍历dict的所有key,并递归删除value为None的key
:param a_dict:
:param attr_name_when_value_is_obj
:return:
"""
if a_dict is None:
return None
elif not isinstance(a_dict, dict):
raise ValueError('参数类型不是字典')
result = dict()
for tmp_key, tmp_value in a_dict.items():
if tmp_value is None:
continue
elif isinstance(tmp_value, dict):
result[tmp_key] = cls.dict_after_remove_key_of_none_value(tmp_value)
elif isinstance(tmp_value, list):
result[tmp_key] = cls.list_after_remove_key_of_none_value(tmp_value)
elif isinstance(tmp_value, str) or isinstance(tmp_value, int) or isinstance(tmp_value, float) or isinstance(
tmp_value, bool):
result[tmp_key] = tmp_value
else:
tmp_value = cls.dict_after_remove_key_of_none_value(
cls.gen_dict_for_a_obj(tmp_value, attr_name_when_value_is_obj))
if tmp_value is not None:
result[tmp_key] = tmp_value
return result
@classmethod
def list_after_remove_key_of_none_value(cls, a_list: Union[list, tuple],
attr_name_when_value_is_obj: str = '__dict__') -> Optional[list]:
r"""
遍历list中每个dict的所有key,并递归删除value为None的key,list中的
:param a_list:
:param attr_name_when_value_is_obj
:return:
"""
if a_list is None:
return None
elif not isinstance(a_list, (list, tuple)):
raise ValueError('参数类型不是list或元组')
result = list()
for tmp_value in a_list:
if tmp_value is None \
or isinstance(tmp_value, str) \
or isinstance(tmp_value, int) \
or isinstance(tmp_value, float) \
or isinstance(tmp_value, bool):
result.append(tmp_value)
elif isinstance(tmp_value, dict):
result.append(cls.dict_after_remove_key_of_none_value(tmp_value))
elif isinstance(tmp_value, list):
result.append(cls.list_after_remove_key_of_none_value(tmp_value))
else:
tmp_value = cls.dict_after_remove_key_of_none_value(
cls.gen_dict_for_a_obj(tmp_value, attr_name_when_value_is_obj))
result.append(tmp_value)
return result
@classmethod
def list2jsonstr(cls, a_list: Union[list, tuple], value_when_obj_is_none: str = 'null',
sort_keys: bool = False, remove_key_of_none_value: bool = False,
attr_name_when_value_is_obj: str = '__dict__') -> str:
r"""
:param a_list:
:param value_when_obj_is_none:
:param sort_keys:
:param remove_key_of_none_value
:param attr_name_when_value_is_obj
:return:
"""
if a_list is None:
return value_when_obj_is_none
elif not isinstance(a_list, (list, tuple)):
raise ValueError('参数类型不是list或元组')
if remove_key_of_none_value:
# 遍历dict的所有key,并递归删除value为None的key
a_list = cls.list_after_remove_key_of_none_value(a_list, attr_name_when_value_is_obj)
sort_keys = True if sort_keys else False
return json.dumps(a_list, ensure_ascii=False,
default=lambda o: cls.gen_dict_for_a_obj(o, attr_name_when_value_is_obj),
sort_keys=sort_keys,
separators=(',', ':'))
def generate_id_16(self) -> str:
r"""
生成一个 16 位的唯一数字编码
:return:
"""
# 产生10000 - 99999 的随机数
r1 = 10000 + random.choice(range(90000))
tmp_time = str(time.time()).replace(".", "")
length_of_tmp_time = len(tmp_time)
while length_of_tmp_time < 13:
tmp_time = str(time.time()).replace(".", "")
length_of_tmp_time = len(tmp_time)
return self.join_str(r1, tmp_time[length_of_tmp_time - 13:length_of_tmp_time - 2])
def generate_id_12(self) -> str:
r"""
生成一个 12 位的唯一数字编码
:return:
"""
# 产生10000 - 99999 的随机数
r1 = 10000 + random.choice(range(90000))
tmp_time = str(time.time()).replace(".", "")
length_of_tmp_time = len(tmp_time)
while length_of_tmp_time < 9:
tmp_time = str(time.time()).replace(".", "")
length_of_tmp_time = len(tmp_time)
return self.join_str(r1, tmp_time[length_of_tmp_time - 9:length_of_tmp_time - 2])
def get_cookies_dict_from_request(self, request, decoding: bool = True) -> dict:
r"""
使用比较原始的方法将request.headers.get('Cookie')中的内容放到一个dict[string,list[string]]结构中
:param request:
:param decoding:
:return:
"""
cookies_str: str = request.headers.get('Cookie')
if not cookies_str:
return dict()
all_cookies_list: list = cookies_str.split(';')
result_dict = dict()
for tmp_cookie_str in all_cookies_list:
tmp_cookie_str_to_list = tmp_cookie_str.split('=')
tmp_cookie_key = tmp_cookie_str_to_list[0].lstrip()
tmp_cookie_value = Utility.list_join_to_str(tmp_cookie_str_to_list[1:], separator_str='=')
if decoding:
tmp_cookie_key = urllib.parse.unquote(tmp_cookie_key, encoding='utf-8')
tmp_cookie_value = urllib.parse.unquote(tmp_cookie_value, encoding='utf-8')
tmp_original_value_list = result_dict.get(tmp_cookie_key)
if tmp_original_value_list is None:
tmp_original_value_list = list()
result_dict[tmp_cookie_key] = tmp_original_value_list
tmp_original_value_list.append(tmp_cookie_value)
return result_dict
def get_bearer_str_from_request(self, request):
r"""
从request的Header Bearer中获取bearer字符串
:param request:
:return:
"""
bearer_str: str = request.headers.get('authorization')
if not bearer_str:
return None
else:
start_str = 'Bearer '
if not bearer_str.startswith(start_str):
return None
else:
return bearer_str[len(start_str):]
def is_mobile_number(self, mobile_str: str) -> bool:
r"""
检查字符串是否是手机格式
:param mobile_str
"""
return re.match('^1\\d{10}$', MyStringUtils.to_str(mobile_str)) is not None
def is_email_address(self, email_str: str) -> bool:
r"""
检查字符串是否是邮箱格式
:param email_str
"""
return re.match(
'^([a-zA-Z0-9_\\-\\.]+)@((\\[[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.)|(([a-zA-Z0-9\\-]+\\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\\]?)$',
MyStringUtils.to_str(email_str)) is not None
def deep_dict_copy(self, a_dict: dict) -> dict:
r"""
:param a_dict:
:return:
"""
result: dict = a_dict.copy()
for tmp_key, tmp_value in result.items():
if isinstance(tmp_value, dict):
result[tmp_key] = self.deep_dict_copy(tmp_value)
if isinstance(tmp_value, list):
result[tmp_key] = self.deep_list_copy(tmp_value)
return result
def deep_list_copy(self, a_list: list) -> list:
r"""
:param a_list:
:return:
"""
result: list = a_list.copy()
for tmp_index, tmp_value in enumerate(result):
if isinstance(tmp_value, dict):
result[tmp_index] = self.deep_dict_copy(tmp_value)
if isinstance(tmp_value, list):
result[tmp_index] = self.deep_list_copy(tmp_value)
return result
def do_notice_admin(self, **kwargs) -> bool:
r"""
通知管理员
异步的方式发送,总是返回true
:key subject 标题,默认为None
:key notice_content 内容,不能为空
:key admin_list 接收人,使用app_config.admin_list 提供
:key runtime_env 使用app_config.runtime_env 提供
:key send_email_config 使用app_config.send_email_config 提供
"""
admin_list = kwargs.get("admin_list")
if not admin_list:
return True
runtime_env = kwargs.get("runtime_env")
send_email_config = kwargs.get("send_email_config")
subject = kwargs.get("subject")
notice_content = kwargs.get("notice_content")
def run():
real_subject = self.join_str(subject, '(运行时环境:', runtime_env, ')')
send_notice_utils = SendNoticeUtils(send_email_config=send_email_config)
send_notice_utils.do_send_email(to_mails=admin_list, subject=real_subject,
notice_content=notice_content)
thread.start_new_thread(run, ())
return True
def common_operation(self, retry_times_when_exception, operation_func, on_operation_failed, on_operation_completed,
*args,
**kwargs):
r"""
操作失败后重试的通用流程
:key retry_times_when_exception
:key operation_func
:key on_operation_completed
:key on_operation_failed
:return:
"""
has_attempt_times = 0
encountered_exception: Optional[BaseException] = None
result: Optional[bool] = None
while has_attempt_times <= retry_times_when_exception:
# 按需重新初始化jedisPool
# 即上次循环中如果发生了异常,则在此次尝试执行操作前先重新初始化
if encountered_exception is not None:
try:
on_operation_failed()
except BaseException as e:
# 如果重新初始化失败,直接结束并抛出异常
encountered_exception = e
break
# 尝试执行操作
try:
result = operation_func(*args, **kwargs)
except AppRuntimeException as e:
encountered_exception = e
break
except BaseException as e:
encountered_exception = e
# 如果没有遇到异常,直接结束循环
if not encountered_exception:
break
# 尝试次数+1
has_attempt_times += 1
on_operation_completed()
if not encountered_exception:
return result
else:
raise encountered_exception
def get_container_name_from_container_image(self, container_image: str):
r"""
从形如 abc.com/istio/tgdevops/tgdevopspyservice:prod-1.0 的uri中提取 tgdevopspyservice
:param container_image:
:return:
"""
sections_of_image = container_image.split(":")
if len(sections_of_image) == 1:
str_contains_name: str = container_image
else:
str_contains_name: str = self.list_join_to_str(sections_of_image[0:len(sections_of_image) - 1],
separator_str="/")
sections_of_image = str_contains_name.split("/")
return sections_of_image[len(sections_of_image) - 1]
def dict_update(self, to_update_dict, update_dict):
r"""
按深度更新字典,尽量保留原字典的内容
列表中不想更新的位置要跳过去需要提供占位符,使用字符串 __placeholder_for_update__ 表示
:param to_update_dict: 被更新的字典
:param update_dict: 提供更新内容的字典
:return:
"""
for key, value in update_dict.items():
original_value = to_update_dict[key]
# 如果更新字典的值不是list,也不是字典,则直接设置更新字典中的内容
# 或者如果原字典中不包含这个key,则直接设置更新字典中的内容
# 或者如果原字典中包含这个key,但是值不是list,也不是字典,则直接设置更新字典中的内容
# 或者原字典中的值的类型和更新字典中的值的类型不一致,则直接设置更新字典中的内容
if (not isinstance(value, dict) and not isinstance(value, list)) \
or key not in to_update_dict \
or (not isinstance(original_value, dict) and not isinstance(original_value, list)) \
or type(original_value) != type(value):
to_update_dict[key] = value
elif isinstance(value, dict):
self.dict_update(original_value, value)
else:
self.list_update(original_value, value)
def list_update(self, to_update_list, update_list):
r"""
按深度更新字典,尽量保留原字典的内容,
列表更新和字典不一样,不想更新的位置要跳过去需要提供占位符,使用字符串 __placeholder_for_update__ 表示
:param to_update_list: 被更新的列表
:param update_list: 提供更新内容的列表,不想更新的index的值使用 __placeholder_for_update__ 表示
:return:
"""
len_of_to_update_list = len(to_update_list)
for index, value in enumerate(update_list):
if index >= len_of_to_update_list:
return
if value == "__placeholder_for_update__":
continue
original_value = to_update_list[index]
# 如果更新列表的值不是list,也不是字典,则直接设置更新字典中的内容
# 或者如果原字典中不包含这个key,则直接设置更新字典中的内容
# 或者如果原字典中包含这个key,但是值不是list,也不是字典,则直接设置更新字典中的内容
# 或者原字典中的值的类型和更新字典中的值的类型不一致,则直接设置更新字典中的内容
if (not isinstance(value, dict) and not isinstance(value, list)) \
or (not isinstance(original_value, dict) and not isinstance(original_value, list)) \
or type(original_value) != type(value):
to_update_list[index] = value
elif isinstance(value, dict):
self.dict_update(original_value, value)
else:
self.list_update(original_value, value)
def replace_placeholder_in_str(self, a_str: str, **kwargs):
r"""
替换字符串中的形如正则表达式${abc}的值为kwargs中的值,如果kwargs中不包含abc,则不进行替换
:param a_str:
:return:
"""
def _get_value_for_matched_key(matched):
key_in_kwargs = matched.group(1)
if key_in_kwargs in kwargs:
result2 = kwargs[key_in_kwargs]
if kwargs is not None:
result2 = str(result2)
else:
result2 = matched.group()
return result2
result = re.sub("\"\\${{int:(.+?)}}\"", _get_value_for_matched_key, a_str)
return re.sub("\\${{(.+?)}}", _get_value_for_matched_key, result)
def generate_random_str(self, randomlength: int = 16) -> str:
"""
生成一个指定长度的随机字符串,其中
string.digits=0123456789
string.ascii_letters=abcdefghigklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ
"""
str_list = [random.choice(string.digits + string.ascii_letters + r"""!@#$%^&*()""") for _
in range(randomlength)]
random_str = ''.join(str_list)
return random_str
def check_resp_status_code_and_content(
self, action, detail_info, resp, is_dict_for_resp_text: bool = True,
expect_http_status_code: int = 200,
**kwargs
) -> Union[Dict, List]:
r"""
通用的检查http请求返回的resp的状态码是否是200,且返回内容是否是json格式,并最终返回resp.text表示的字典
:key action
:key detail_info
:key resp
:key is_dict_for_resp_text 默认为True
:key expect_http_status_code 默认为200
:key other_expect_http_status_codes 其他期望的status_code值
:key action_desc
:return:
"""
# 判断状态码
other_expect_http_status_codes = kwargs.get("other_expect_http_status_codes")
if other_expect_http_status_codes is None:
other_expect_http_status_codes = list()
other_expect_http_status_codes.append(expect_http_status_code)
action_desc = kwargs.get("action_desc")
if not action_desc:
action_desc = action
if resp.status_code not in other_expect_http_status_codes:
raise AppRuntimeException(Utility.join_str(
action_desc,
f"失败,返回的状态码不是{other_expect_http_status_codes}"),
Utility.join_str(
action,
f"返回的状态码不是{other_expect_http_status_codes},statusCode=",
resp.status_code, " ",
detail_info))
# 判断返回内容
result_str = resp.text
if is_dict_for_resp_text:
resp_obj = Utility.jsonstr2dict(result_str)
else:
resp_obj = Utility.jsonstr2list(result_str)
if resp_obj is None:
raise AppRuntimeException(
Utility.join_str(action_desc, "失败,返回的内容不是json格式"),
Utility.join_str(
action,
f"返回的内容不是json格式,content={result_str}",
" ",
detail_info
)
)
return resp_obj
def check_resp_status_code_and_code(
self,
action,
detail_info,
resp,
expect_code_value: str = ConstResponseCode.CODE_OK,
other_expect_code_values: list = None,
b_check_data_in_app_response: bool = True,
**kwargs
):
r"""
通用的检查http请求返回的resp的状态码是否是200,且返回内容为iam标准response,且其中的code为OK,且其中的data字段不为空,
并最终返回data字段
:key action:
:key detail_info:
:key resp:
:key expect_code_value:
:key other_expect_code_values:
:key b_check_data_in_app_response:
:key key_for_data:
:key key_for_message:
:key key_for_code:
:key action_desc:
:return:
"""
return self.check_resp_status_code_and_code_and_return_app_response(
action,
detail_info,
resp,
expect_code_value=expect_code_value,
other_expect_code_values=other_expect_code_values,
b_check_data_in_app_response=b_check_data_in_app_response,
**kwargs
).data
def check_resp_status_code_and_code_and_return_app_response(
self,
action,
detail_info,
resp,
expect_code_value: str = ConstResponseCode.CODE_OK,
other_expect_code_values: list = None,
b_check_data_in_app_response: bool = True,
**kwargs
) -> AppResponse:
r"""
通用的检查http请求返回的resp的状态码是否是200,且返回内容为iam标准response,且其中的code为OK,且其中的data字段不为空,
并最终返回data字段
:key action:
:key detail_info:
:key resp:
:key expect_code_value:
:key other_expect_code_values:
:key b_check_data_in_app_response:
:key key_for_data:
:key key_for_message:
:key key_for_code:
:key action_desc:
:return:
"""
action_desc = kwargs.get("action_desc")
if not action_desc:
action_desc = action
resp_dict = self.check_resp_status_code_and_content(action, detail_info, resp, action_desc=action_desc)
# 转换成标准的 AppResponse
key_for_code = kwargs.get("key_for_code")
key_for_data = kwargs.get("key_for_data")
key_for_message = kwargs.get("key_for_message")
app_response = AppResponse.from_dict(resp_dict, key_for_data=key_for_data, key_for_msg=key_for_message,
key_for_code=key_for_code)
if other_expect_code_values is None:
other_expect_code_values = list()
other_expect_code_values.append(expect_code_value)
if app_response.code not in other_expect_code_values:
raise AppRuntimeException(
Utility.join_str(action_desc, "失败,返回的code=", app_response.code, " message=",
app_response.message),
Utility.join_str(action, "返回的code=", app_response.code, " message=",
app_response.message, " ", detail_info))
iam_response_data = app_response.data
# 如果未获取到权限验证结果信息,直接抛出异常
if not iam_response_data and b_check_data_in_app_response:
raise AppRuntimeException(Utility.join_str(action_desc, "失败,返回的内容不包含data数据"),
Utility.join_str(action, "返回的内容不包含data数据,content=",
Utility.dict2jsonstr(resp_dict), " ", detail_info))
return app_response
def parse_crn(self, crn: str, *crn_list) -> CrnParseResult:
r"""
解析格式为 crn:ucs::{product_code}:{region_code}:{account_id}:first/{first_id}/second/{second_id} 的crn表达式
从中解析出 product_code,region_code,acct_id,first_id,second_id,third_id
:param crn:
:param crn_list: 多个crn,解析第一个不为空的crn
"""
if not crn:
for tmp_crn in crn_list:
if tmp_crn:
crn = tmp_crn
break
if not crn:
raise ValueError(f"crn为空")
colon_parts = crn.split(":")
if len(colon_parts) < 7:
raise ValueError(f"{crn}不是有效的crn表达式")
product_code = colon_parts[3]
region_code = colon_parts[4]
acct_id = colon_parts[5]
base_crn = ":".join(colon_parts[0:6])
res_part = colon_parts[6]
res_parts = res_part.split("/")
len_parts = len(res_parts)
first_id_type = None
first_id = None
second_id_type = None
second_id = None
third_id_type = None
third_id = None
other_id_types = list()
other_ids = list()
first_crn = None
second_crn = None
third_crn = None
other_crns = list()
if len_parts >= 2:
first_id_type = res_parts[0]
first_id = res_parts[1]
tmp_res_part = "/".join(res_parts[0:2])
first_crn = ":".join([base_crn, tmp_res_part])
if len_parts >= 4:
second_id_type = res_parts[2]
second_id = res_parts[3]
tmp_res_part = "/".join(res_parts[0:4])
second_crn = ":".join([base_crn, tmp_res_part])
if len_parts >= 6:
third_id_type = res_parts[4]
third_id = res_parts[5]
tmp_res_part = "/".join(res_parts[0:6])
third_crn = ":".join([base_crn, tmp_res_part])
if len_parts >= 8:
for index in range(7, len_parts):
if index % 2 != 0:
other_ids.append(res_parts[index])
tmp_res_part = "/".join(res_parts[0:index + 1])
other_crns.append(":".join([base_crn, tmp_res_part]))
else:
other_id_types.append(res_parts[index])
return CrnParseResult(
product_code=product_code,
region_code=region_code,
acct_id=acct_id,
first_id_type=first_id_type,
first_id=first_id,
second_id_type=second_id_type,
second_id=second_id,
third_id_type=third_id_type,
third_id=third_id,
other_id_types=other_id_types,
other_ids=other_ids,
first_crn=first_crn,
second_crn=second_crn,
third_crn=third_crn,
other_crns=other_crns,
)
def gen_crn_expression(self, product_code: str, region_code: str, account_id: str, res_part: str) -> str:
r"""
生成crn表达式, 示例 crn:ucs::cfc:gz-tst:12345:func/helloworld
:param product_code: 例如 cfc
:param region_code: 例如 gz-tst
:param account_id:
:param res_part:资源部分的描述字符串,例如 func/helloworld
"""
region_code = region_code or ""
return f"crn:ucs::{product_code}:{region_code}:{account_id}:{res_part}"
def rmdir_or_file(self, file_path: str):
r"""
根据操作系统执行删除文件夹/文件
:param file_path:
"""
if not os.path.exists(file_path):
return
if os.path.isdir(file_path):
# 执行删除文件夹操作
if platform.system() == 'Windows':
subprocess.call(f"rmdir /S /Q {file_path}", shell=True)
else:
subprocess.call(f"rm -rf {file_path}", shell=True)
elif os.path.isfile(file_path):
# 执行删除文件操作
os.remove(file_path)
else:
# 特殊文件,不执行操作
self.logger.warning(
f"执行删除操作时发现{file_path} is a special file(socket,FIFO,device file),不执行删除操作")
def read_file_content(self, file_path) -> str:
"""
读取文件内容
:param file_path: 文件路径
:return: str -> file-content
"""
with open(file_path, 'r') as f: # 打开一个文件,必须是'rb'模式打开
return f.read()
def yaml_load(self, yaml_str: str) -> Union[dict, str]:
r"""
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
"""
dict_list = self.yaml_load_list(cfg_content=yaml_str)
if dict_list:
return dict_list[0]
else:
return dict()
def yaml_load_list(self, path: str = None, cfg_content: str = None) -> List[Union[Dict, str]]:
"""
将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据,每个dict放到list中返回
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
:param path: 文件路径
:param cfg_content:
:return: list -> []
"""
if path:
cfg = self.read_file_content(file_path=path)
else:
cfg = cfg_content or ""
yaml_generator = yaml.safe_load_all(cfg) # 将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据
if yaml_generator is None:
return list()
# 转成dict并保存到list中
yaml_list = list()
for one_yaml_generator in yaml_generator:
if one_yaml_generator is None:
continue
yaml_list.append(json.loads(json.dumps(one_yaml_generator)))
return yaml_list
def yaml_dump_list_to_str(self, source_list: List[Dict]) -> str:
r"""
将字典列表序列化成为yaml字符串
"""
return yaml.safe_dump_all(documents=source_list, encoding='utf-8', allow_unicode=True).decode('utf-8')
def yaml_dump_list_to_file(self, source_list: List[Dict], file_path: str):
r"""
将字典列表序列化成为yaml字符串
"""
with open(file_path, 'w', encoding='utf-8') as f: # 打开一个文件,必须是'rb'模式打开
yaml.safe_dump_all(documents=source_list, stream=f, encoding='utf-8', allow_unicode=True)
def dict_pop_key(self, source_dict: dict, key_name: str):
r"""
改进的移除dict的key,如果key不存在,则返回None,否则返回原始的 dict.pop(key)
:param source_dict:
:param key_name:
"""
return source_dict.pop(key_name) if key_name in source_dict else None
def gen_user_info(self, login_user: LoginedUserInfo, output_type: str = "str") -> Union[str, dict]:
r"""
生成 {"user_id":"","user_name":""}格式的字典或字符串
:param login_user:
:param output_type: str or dict, 输出格式,默认为 str
"""
output_type = output_type or "str"
result_dict = {
"user_id": str(login_user.user_id),
"user_name": login_user.user_name
}
if output_type == "str":
return self.dict2jsonstr(result_dict)
else:
return result_dict
def get_content_after_render(
self,
dir_path: str = "./resource/",
encoding: str = "utf-8",
template_name: str = None,
render_dict: dict = None,
**kwargs
) -> str:
r"""
获取jinja2 template渲染后的文件内容
:param dir_path: 被渲染文件所在目录,默认为 ./resource/
:param encoding: 获取文件内容编码,默认为 utf-8
:param template_name: 模板文件名称
:param render_dict: 渲染模板用的字典
:key variable_start_string:
:key variable_end_string:
"""
dir_path = dir_path or "./resource/"
encoding = encoding or "utf-8"
render_dict = render_dict or dict()
variable_start_string = kwargs.get("variable_start_string") or VARIABLE_START_STRING
variable_end_string = kwargs.get("variable_end_string") or VARIABLE_END_STRING
env = Environment(
loader=FileSystemLoader(dir_path, encoding=encoding),
variable_start_string=variable_start_string,
variable_end_string=variable_end_string,
undefined=StrictUndefined,
) # 创建文件加载器对象
template = env.get_template(template_name) # 获取一个模板文件
return template.render(render_dict) # 渲染
def gen_ip_port_str(self, schema: Optional[str] = "http", num_list: list = None, port: Optional[int] = None):
r"""
生成url格式字符串,防止sonar检查
:param schema: 默认为http,None时不计入字符串内容
:param num_list: IP地址数字
:param port: 端口数字
"""
result_str = f"{schema}://" if schema else ""
str_list = [str(num) for num in num_list]
result_str += ".".join(str_list)
if port is not None:
result_str += f":{port}"
return result_str
def get_md5_of_string(self, src: str) -> str:
"""
获取字符串的md5值
:param src:
:return:
"""
md1 = hashlib.md5() # 创建一个md5算法对象
md1.update(src.encode('UTF-8'))
return md1.hexdigest()
def get_md5_of_file(self, filepath: str, read_byte_once: int = 8096) -> str:
"""
获取文件的md5值
:param filepath:
:param read_byte_once:
:return:
"""
if not os.path.isfile(filepath):
return ""
myhash = hashlib.md5() # 创建一个md5算法对象
with open(filepath, 'rb') as f: # 打开一个文件,必须是'rb'模式打开
while True:
b = f.read(read_byte_once) # 由于是一个文件,每次只读取固定字节
if not b:
break
# 当读取内容不为空时对读取内容进行update
myhash.update(b)
return myhash.hexdigest()
def get_md5_of_dir(self, dirpath: str) -> str:
"""
获取文件夹的md5值
:param dirpath:
:return:
"""
if not os.path.isdir(dirpath):
return ""
dir_path_for_md5_file = tempfile.mkdtemp()
md5_file = os.path.join(dir_path_for_md5_file, "tmp.md5")
with open(md5_file, 'w') as outfile:
for root, subdirs, files in os.walk(dir_path_for_md5_file):
for file in files:
filefullpath = os.path.join(root, file)
md5 = self.get_md5_of_file(filefullpath)
outfile.write(md5)
val = self.get_md5_of_file(md5_file)
self.rmdir_or_file(dir_path_for_md5_file)
return val
def get_md5_of_file_or_dir(self, filepath: str) -> str:
r"""
获取文件/文件夹的md5值
:param filepath:文件/文件夹的路径
"""
if os.path.isfile(filepath):
return self.get_md5_of_file(filepath)
elif os.path.isdir(filepath):
return self.get_md5_of_dir(filepath)
else:
return ""
def base64_decode_with_padding(self, payload: Union[str, bytes]) -> str:
r"""
base64解码,如果payload字节数不是4的倍数,会使用=补足
:param payload: 字符串或bytes
"""
if isinstance(payload, str):
byte_payload = payload.encode("utf-8")
else:
byte_payload = payload
missing_padding = 4 - len(byte_payload) % 4
if missing_padding:
byte_payload += b'=' * missing_padding
b_result = base64.b64decode(byte_payload)
return str(b_result, encoding="utf-8")
def parse_jwt_str(self, jwt_str: str) -> str:
r"""
:param jwt_str: jwt字符串
"""
payload = jwt_str.split(".")[1]
return self.base64_decode_with_padding(payload)
def attemp_parse_login_user_info(self, jwt_str: str = None, request: Request = None) -> Optional[LoginedUserInfo]:
r"""
尝试从jwt字符串中解析出登陆用户信息,解析失败返回None
:param jwt_str: jwt字符串
:param request: flask.Request对象
"""
if not jwt_str and request:
jwt_str = request.cookies.get("accessToken") or ""
try:
payload = self.parse_jwt_str(jwt_str) if jwt_str else None
if not payload:
raise AppRuntimeException(
message="解析jwt字符串后获取的payload内容为空",
detail=f"jwt_str= {jwt_str}",
)
payload_dict = self.jsonstr2dict(payload)
if not payload_dict:
raise AppRuntimeException(
message="jwt的payload为空或者不是有效的json字符串",
detail=f"payload= {payload}",
)
return LoginedUserInfo.from_dict(payload_dict, jwt_str)
except BaseException as e:
self.logger.error(e)
exception_tracback = traceback.format_exc()
self.logger.error(exception_tracback)
return None
def retry_operation(
self, operation_func, is_ok_function, is_ok_function_for_exception=None,
operation_func_desc: str = None,
max_execute_times: int = 240, wait_seconds_for_retry: float = 1
):
r"""
通用的重试动作
:param operation_func: 操作函数,请使用 functools.partial 保证传入的函数在执行时不用再传入任何参数
:param is_ok_function: 对函数执行结果的判断函数,
该函数应返回一个Tuple,第一个元素为 bool 类型,表示对操作结果是否认为成功,第二个参数为真正的函数返回结果
:param is_ok_function_for_exception: 如果提供了该函数,则函数执行期间发生异常时使用该函数对异常进行判断,
该函数应返回一个Tuple,第一个元素为 bool 类型,表示发生异常时是否认为操作成功,第二个参数为认为成功时的函数返回结果
:param operation_func_desc: 操作函数描述,默认为 字面值"operation_func"
:param max_execute_times: 默认 240次
:param wait_seconds_for_retry: 默认 1 秒
"""
operation_func_desc = operation_func_desc or "operation_func"
has_executed_times = 0
while True:
# 尝试执行,并获得结果或捕获异常
tmp_result = None
encountered_exception = None
try:
tmp_result = operation_func()
is_ok, tmp_result = is_ok_function(tmp_result)
if is_ok:
return tmp_result
except BaseException as e:
if is_ok_function_for_exception:
is_ok, tmp_result2 = is_ok_function_for_exception(e)
if is_ok:
return tmp_result2
else:
self.logger.error(f"执行{operation_func_desc}时发生异常:{e}")
self.logger.error(traceback.format_exc())
encountered_exception = e
else:
self.logger.error(f"执行{operation_func_desc}时发生异常:{e}")
self.logger.error(traceback.format_exc())
encountered_exception = e
# 记录执行次数+1
has_executed_times += 1
# 判断是否已经达到最大执行次数
if has_executed_times >= max_execute_times > 0:
# 返回最后一次执行的结果或抛出异常
if encountered_exception:
raise encountered_exception
else:
return tmp_result
# 睡眠间隔
time.sleep(wait_seconds_for_retry)
def get_dict_from_dict_and_set_when_none(self, source_dict: dict, key_name: str) -> dict:
r"""
从字典中获取字典,如果指定的key不存在,则设置key对应的对象为一个空的字典
:param source_dict:
:param key_name:
"""
result_dict = source_dict.get(key_name)
if result_dict is None:
result_dict = dict()
source_dict[key_name] = result_dict
return result_dict
def get_list_from_dict_and_set_when_none(self, source_dict: dict, key_name: str) -> list:
r"""
从字典中获取字典,如果指定的key不存在,则设置key对应的对象为一个空的字典
:param source_dict:
:param key_name:
"""
result_list = source_dict.get(key_name)
if result_list is None:
result_list = list()
source_dict[key_name] = result_list
return result_list
def serialized(self, source_obj) -> str:
r"""
序列化为字符串,使用 pickle.dump 然后 base64编码, 然后以 utf-8 解码为字符串
"""
obj_src_bytes = pickle.dumps(source_obj)
return base64.b64encode(obj_src_bytes).decode("utf-8")
def reverse_serialized(self, source_str: str):
r"""
反序列化,将字符串以utf-8编码为bytes,然后base64解码,然后 pickle.loads为对象
"""
obj_src_bytes = base64.b64decode(source_str.encode("utf-8"))
return pickle.loads(obj_src_bytes)
def parse_cpu_value_to_m(
self,
source_str: str,
parse_empty_to_0: bool = False,
) -> int:
"""
根据 10m 或 10 这样的字符串计算 cpu 有多少 m
"""
source_str_desc = source_str or "空字符串"
if not source_str and parse_empty_to_0:
return 0
elif source_str.endswith("m"):
return int(source_str[0:-1])
elif source_str.isdigit(): # 全部是数字,则转换为m需要乘1000
return int(float(source_str) * 1000)
else:
failed_reason = f"无法将{source_str_desc}解析为单位为m的CPU核数"
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=failed_reason,
)
def parse_mem_value_to_b(
self,
source_str: str,
parse_empty_to_0: bool = False,
) -> int:
"""
根据 100Mi 或 10Gi 这样的字符串计算 mem 有多少字节
"""
source_str_desc = source_str or "空字符串"
if not source_str and parse_empty_to_0:
return 0
elif source_str.endswith("Ki"):
return int(float(source_str[0:-2]) * 1024 ** 1)
elif source_str.endswith("Mi"):
return int(float(source_str[0:-2]) * 1024 ** 2)
elif source_str.endswith("Gi"):
return int(float(source_str[0:-2]) * 1024 ** 3)
elif source_str.endswith("Ti"):
return int(float(source_str[0:-2]) * 1024 ** 4)
elif source_str.endswith("Pi"):
return int(float(source_str[0:-2]) * 1024 ** 5)
elif source_str.endswith("Ei"):
return int(float(source_str[0:-2]) * 1024 ** 6)
elif source_str.endswith("k"):
return int(float(source_str[0:-1]) * 1000 ** 1)
elif source_str.endswith("M"):
return int(float(source_str[0:-1]) * 1000 ** 2)
elif source_str.endswith("G"):
return int(float(source_str[0:-1]) * 1000 ** 3)
elif source_str.endswith("T"):
return int(float(source_str[0:-1]) * 1000 ** 4)
elif source_str.endswith("P"):
return int(float(source_str[0:-1]) * 1000 ** 5)
elif source_str.endswith("E"):
return int(float(source_str[0:-1]) * 1000 ** 6)
elif source_str.isdigit(): # 全部是数字则单位为B
return int(source_str)
else:
failed_reason = f"无法将{source_str_desc}解析为内存字节数量"
raise AppException(
code=ConstResponseCode.CODE_MISSING_PARAMETER,
message=failed_reason,
)
def format_time_to_utc_str(self, param_time: Union[int, float, str, datetime]) -> Optional[str]:
r"""
将时间格式化成UTC字符串,格式为标准时区的 %Y-%m-%dT%H:%M:%SZ
:param param_time: 可以是unix时间戳(秒数),也可以是本地日期的字符串,或者是datetime类型的日期
"""
if param_time is None:
return None
if isinstance(param_time, (int, float)):
parsed_utctime = datetime.utcfromtimestamp(param_time)
return parsed_utctime.strftime('%Y-%m-%dT%H:%M:%SZ')
elif isinstance(param_time, datetime):
parsed_utctime = datetime.utcfromtimestamp(param_time.timestamp())
return parsed_utctime.strftime('%Y-%m-%dT%H:%M:%SZ')
elif isinstance(param_time, str):
if not param_time:
return None
parsed_time = self.parse_datestr(param_time)
if parsed_time is None:
raise ValueError(f"param_time不是有效的时间字符串:{param_time}")
parsed_utctime = datetime.utcfromtimestamp(parsed_time.timestamp())
return parsed_utctime.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
raise ValueError(f"param_time不能解析为时间:不支持的类型:{param_time.__class__}")
def format_time_to_unix_timestamp_seconds(self, param_time: Union[int, float, str, datetime]) -> Optional[float]:
r"""
将时间格式化成unix时间戳(秒数)
:param param_time: 可以是unix时间戳(秒数),也可以是本地日期的字符串,或者是datetime类型的日期
"""
if param_time is None:
return None
if isinstance(param_time, (int, float)):
return param_time
elif isinstance(param_time, datetime):
return param_time.timestamp()
elif isinstance(param_time, str):
if not param_time:
return None
parsed_time = self.parse_datestr(param_time)
if parsed_time is None:
raise ValueError(f"param_time不是有效的时间字符串:{param_time}")
return parsed_time.timestamp()
else:
raise ValueError(f"param_time不能解析为unix时间戳:不支持的类型:{param_time.__class__}")
def get_object_from_obj_by_paths(self, source_obj, key_paths: List[str] = None, key_path_str: str = None):
r"""
按照key的路径依次获取指定路径序列的key对应的字典或列表,如果中途key miss,或index超出范围,则返回None,
:param source_obj: 列表或字典
:param key_paths: 优先级比 key_path_str 高
:param key_path_str: 格式为 components.ingressGateways.0.k8s.resources
"""
if not key_paths:
key_path_str = key_path_str or ""
key_paths = [int(x) if x.isdigit() else x for x in key_path_str.strip(".").split(".")]
result_obj = source_obj
for tmp_key in key_paths:
if isinstance(tmp_key, int):
if len(result_obj) > tmp_key:
result_obj = result_obj[tmp_key]
else:
result_obj = None
else:
result_obj = result_obj.get(tmp_key)
if result_obj is None:
return result_obj
return result_obj
def load_file_as_dict(self, filepath: str) -> dict:
r"""
判断文件后缀名, 如果是 .yaml 或 .yml 则按 yaml 方式读取,
文件内容不合规,则抛出异常
"""
with open(filepath, 'r', encoding="utf-8") as dict_file:
conf_content = dict_file.read()
# 无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
dict_content = self.yaml_load(conf_content)
if isinstance(dict_content, str):
raise ValueError(f"{filepath}的内容不是有效的json或yaml字符串")
return dict_content
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment