Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
qiuqunfeng
common_pkg
Commits
11b2601e
Commit
11b2601e
authored
Mar 13, 2025
by
qunfeng qiu
Browse files
Initial commit
parents
Changes
148
Show whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
5589 additions
and
0 deletions
+5589
-0
cucc_common_pkg/loginhandle.py
cucc_common_pkg/loginhandle.py
+156
-0
cucc_common_pkg/ludq_utils/README.md
cucc_common_pkg/ludq_utils/README.md
+80
-0
cucc_common_pkg/ludq_utils/__init__.py
cucc_common_pkg/ludq_utils/__init__.py
+10
-0
cucc_common_pkg/ludq_utils/aes_utils.py
cucc_common_pkg/ludq_utils/aes_utils.py
+122
-0
cucc_common_pkg/ludq_utils/app_exception.py
cucc_common_pkg/ludq_utils/app_exception.py
+124
-0
cucc_common_pkg/ludq_utils/app_response.py
cucc_common_pkg/ludq_utils/app_response.py
+158
-0
cucc_common_pkg/ludq_utils/base_const.py
cucc_common_pkg/ludq_utils/base_const.py
+40
-0
cucc_common_pkg/ludq_utils/base_db_service.py
cucc_common_pkg/ludq_utils/base_db_service.py
+466
-0
cucc_common_pkg/ludq_utils/base_redis_service.py
cucc_common_pkg/ludq_utils/base_redis_service.py
+579
-0
cucc_common_pkg/ludq_utils/cert_utils.py
cucc_common_pkg/ludq_utils/cert_utils.py
+483
-0
cucc_common_pkg/ludq_utils/common_app_config.py
cucc_common_pkg/ludq_utils/common_app_config.py
+194
-0
cucc_common_pkg/ludq_utils/common_base_webservice.py
cucc_common_pkg/ludq_utils/common_base_webservice.py
+794
-0
cucc_common_pkg/ludq_utils/common_base_webservice_for_stream.py
...ommon_pkg/ludq_utils/common_base_webservice_for_stream.py
+64
-0
cucc_common_pkg/ludq_utils/logined_user.py
cucc_common_pkg/ludq_utils/logined_user.py
+94
-0
cucc_common_pkg/ludq_utils/md5_utils.py
cucc_common_pkg/ludq_utils/md5_utils.py
+33
-0
cucc_common_pkg/ludq_utils/rsa_utils.py
cucc_common_pkg/ludq_utils/rsa_utils.py
+98
-0
cucc_common_pkg/ludq_utils/send_notices_utils.py
cucc_common_pkg/ludq_utils/send_notices_utils.py
+317
-0
cucc_common_pkg/ludq_utils/util_methods.py
cucc_common_pkg/ludq_utils/util_methods.py
+390
-0
cucc_common_pkg/ludq_utils/utils_base.py
cucc_common_pkg/ludq_utils/utils_base.py
+1377
-0
cucc_common_pkg/ludq_utils/utils_get_cluster_info/__init__.py
..._common_pkg/ludq_utils/utils_get_cluster_info/__init__.py
+10
-0
No files found.
cucc_common_pkg/loginhandle.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
copy
import
json
import
uuid
from
functools
import
wraps
from
flask
import
current_app
,
request
from
.cfginfohandle
import
CfgInfoHandle
from
.globalconst
import
CookiesConstKey
,
FlaskConfigIDConst
,
RedisConnConst
,
RedisDbNumConst
,
GlobalConst
,
\
GlobalRetKeyConst
,
RedisInfoKeysConst
from
.globalerror
import
CommonError
from
.my_baseexception
import
MyBaseException
from
.my_servicehandle
import
ServiceHandle
from
.redishanlde
import
RedisHandle
class
LoginHandle
:
# 过滤规则
FILTER_RULE_NAME
=
"filter_name"
# 过滤具体值
FILTER_RULE_VALUE
=
"filter_val"
# 精确匹配
FILTER_EXACT
=
"filter_exact"
# 正则匹配
FILTER_REG
=
"filter_regular"
def
__init__
(
self
,
flask_app
=
None
,
list_filter_url
:
list
=
None
):
r
"""
登录处理初始化
Args:
flask_app: flask app
list_filter_url: url过滤,不需要被登录检查的 list[dict]
"""
# 初始化,处理是否接入flask
self
.
list_filter_url
=
None
if
list_filter_url
is
not
None
:
self
.
list_filter_url
=
list_filter_url
self
.
app
=
flask_app
if
flask_app
is
not
None
:
self
.
init_app
(
flask_app
)
def
init_app
(
self
,
app
):
r
"""
初始化flask变量,走所有服务之前去验证登录
Args:
app:
Returns:
"""
# app.before_request(self.checklogin)
pass
def
login
(
self
,
login_name
:
str
=
None
,
login_pass
:
str
=
None
,
login_code
:
str
=
None
,
tenant_id
:
str
=
None
,
proc_name
:
str
=
"auth_userlogin_proc"
,
login_srv_url
:
str
=
None
,
redis_info
:
dict
=
None
)
->
tuple
:
R
"""
Args:
login_name: 登录账号
login_pass: 登录密码
login_code: 登录验证码
tenant_id: 租户ID
proc_name: proc名称
login_srv_url: 登录url
redis_info: redis信息
Returns:
"""
# 登录验证
service_url
=
login_srv_url
trans_data
=
{
"arg_tenantid"
:
tenant_id
,
"arg_username"
:
login_name
,
"arg_userpass"
:
login_pass
}
sh
=
ServiceHandle
()
ret_json
:
dict
=
sh
.
call_proc
(
service_url
=
service_url
,
proc_name
=
proc_name
,
arg_dict
=
trans_data
)
# 返回失败
if
sh
.
check_is_success
(
ret_json
)
is
False
or
GlobalRetKeyConst
.
DATAROWS
not
in
ret_json
or
len
(
ret_json
[
GlobalRetKeyConst
.
DATAROWS
])
==
0
:
return
ret_json
,
None
user_info
:
dict
=
ret_json
[
GlobalRetKeyConst
.
DATAROWS
][
0
]
jsessionid
=
str
(
uuid
.
uuid1
()).
replace
(
'-'
,
''
)
dict_kv
=
{
jsessionid
:
json
.
dumps
(
user_info
,
ensure_ascii
=
False
)
}
# 校验成功写入缓存
rh
:
RedisHandle
=
current_app
.
config
[
FlaskConfigIDConst
.
REDIS_CLASS_ID
]
rh
.
set_vals
(
dict_redisinfo
=
redis_info
,
dict_kv
=
dict_kv
)
return
ret_json
,
jsessionid
def
check_login
(
func
):
r
"""
装饰器检测是否登录
Returns:
"""
@
wraps
(
func
)
def
wrapper
(
*
args
,
**
kwargs
):
if_login
=
False
# 检查cookies是否为null 或者 cookies中的sessionid的值是否在redis中存在
if
request
.
cookies
.
keys
()
is
not
None
and
CookiesConstKey
.
LOGIN_SESSTION_KEY
in
request
.
cookies
.
keys
():
if_login
=
True
rh
:
RedisHandle
=
current_app
.
config
[
FlaskConfigIDConst
.
REDIS_CLASS_ID
]
cfg_handle
:
CfgInfoHandle
=
current_app
.
config
[
FlaskConfigIDConst
.
INIT_CONFIG_ID
]
dict_conn
:
dict
=
copy
.
copy
(
cfg_handle
.
get_redis_info
()[
RedisInfoKeysConst
.
REDIS_SLAVE_KEY
])
dict_conn
[
RedisConnConst
.
CONN_DB
]
=
RedisDbNumConst
.
LOGIN_INFO
csm_id
=
request
.
cookies
[
CookiesConstKey
.
LOGIN_SESSTION_KEY
]
dict_key
=
{
csm_id
:
csm_id
}
ret_dict
=
rh
.
get_vals
(
dict_key
,
dict_conn
)
if
csm_id
not
in
ret_dict
:
if_login
=
False
if
if_login
is
True
:
return
func
(
*
args
,
**
kwargs
)
else
:
return
json
.
dumps
(
MyBaseException
.
format_to_standard_dict
(
ret_code
=
GlobalConst
.
RETCODE_COMMON_ERROR
,
ret_val
=
CommonError
.
ERROR_USER_UNLOGIN
),
ensure_ascii
=
False
).
encode
(
'utf-8'
)
return
wrapper
cucc_common_pkg/ludq_utils/README.md
0 → 100644
View file @
11b2601e
# venv 安装
建议使用 venv 虚拟环境 使用pycharm自动生成venv
或者在项目文件夹下执行下面的语句后再在pycharm中设置python解释器为虚拟环境中的解释器
```
shell
# 创建虚拟环境
python3
-m
venv venv
# 激活虚拟环境
source
venv/Scripts/activate
# 或者在windows 下直接使用 venv\Scripts\activate 激活
```
配置好虚拟环境后,再进行依赖包安装
# 依赖包安装
```
shell
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
simplejson
==
3.17.0
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
Flask
==
1.1.2
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
pymysql
==
0.10.1
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
redis
==
3.5.3
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
pytz
==
2021.1
# aes加密使用的库
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
pycryptodome
==
3.9.9
# yaml文件
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
pyyaml
==
5.3.1
# rsa解密
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
rsa
==
4.6
# 数据库连接池
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
DBUtils
==
2.0
# COS操作
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
boto
==
2.49.0
# rarfile操作
pip3
install
-i
http://10.126.154.1:18083/repository/swrdcucc-group/simple
--trusted-host
10.126.154.1
rarfile
==
4.0
# 卸载模块
pip3 uninstall cucc_common_pkg
```
# 发布
```
pip3 install --upgrade setuptools wheel twine
# 如有必要, 先将原先编译生成的目录删掉, 例如 build,dist目录
# setup.py文件同目录命令行下运行
python setup.py sdist bdist_wheel
# 检查打包的文件是否正常
python setup.py install # 安装
# 按照使用方式导入测试,没问题后继续
# 上传
twine upload dist/* --repository-url http://10.126.154.1:18083/repository/swrdcucc-hosted/ -u {wenhx8} -p {dwp}
```
# 项目调试
## 配置环境变量
```
PYTHONUNBUFFERED=1
ENV_APP_CONFIG={"db_config":{"host":"sky-mysql","port":3306,"user":"","passwd":"","database":"","charset":"utf8mb4"},"master_redis_config":{"host":"csm-redis-master.default.svc","port":6379,"password":"","db":"5"},"slave_redis_config":{"host":"csm-redis-slave.default.svc","port":6379,"password":"","db":"5"},"is_debug":false,"is_send_admin_notice":false,"is_send_user_notice":false,"send_email_config":{"mail_host":"hq.smtp.chinaunicom.cn","mail_from_account":"hqs-ioa-cusri@chinaunicom.cn","mail_from_account_dwp":""},"send_sms_config":{"api_server":"https://sms.tg.unicom.local/sms/message/send","sys_name":"","sys_token":""},"admin_list":["ludq1@chinaunicom.cn"],"runtime_env":"DEV","aes_mysql_key":"","rsa_private_key":"","sso_api":"https://sso.dev.tg.unicom.local/sso/v1","iam_api":"https://iam.dev.tg.unicom.local/iam/v2","configcenter_api":"https://configcenter.dev.tg.unicom.local/configcenter/v1","product_vpc_api":"https://vpc.console.dev.tg.unicom.local","product_clb_api":"https://vpc.console.dev.tg.unicom.local","product_cke_api":"https://cke.console.dev.tg.unicom.local","product_csm_api":"https://csm.console.dev.tg.unicom.local","product_ccr_api":"https://ccr.console.dev.tg.unicom.local","product_rds_api":"https://rds.console.dev.tg.unicom.local","product_redis_api":"https://redis.console.tg.unicom.local","product_kafka_api":"https://kafka.console.dev.tg.unicom.local","tianti_api":"https://tianti.dev.tg.unicom.local","ide_apiserver":"","ide_apiserver_secret":"disable","ide_image":""}
ENV_APP_CONFIG_INIT_STYLE=file
ENV_APP_CONFIG_FILE_PATH=C:\Users\supershll\Desktop\app_config.json
```
cucc_common_pkg/ludq_utils/__init__.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
cucc_common_pkg/ludq_utils/aes_utils.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
base64
from
typing
import
Optional
from
Crypto.Cipher
import
AES
class
AesUtilsForMysql
:
r
"""
Mysql AES加密 utils类,不依赖任何其他应用基础类
"""
def
__init__
(
self
,
key
:
str
=
None
):
r
"""
初始化
:param key
"""
self
.
key
:
str
=
key
if
self
.
key
:
self
.
real_aes_key
:
Optional
[
bytes
]
=
self
.
gen_aes_key_by_mysql_key
(
self
.
key
)
else
:
self
.
real_aes_key
:
Optional
[
bytes
]
=
None
def
pkcs7padding_to_encrypt_content
(
self
,
to_encrypt_content
:
str
)
->
str
:
r
"""
明文使用PKCS7填充
最终调用AES加密方法时,传入的是一个byte数组,要求是16的整数倍,因此需要对明文进行处理
:param to_encrypt_content
:return
"""
bytes_length
=
len
(
bytes
(
to_encrypt_content
,
encoding
=
'utf-8'
))
# tips:utf-8编码时,英文占1个byte,而中文占3个byte
padding_times
=
16
-
bytes_length
%
16
# tips:chr(padding)看与其它语言的约定,有的会使用'\0'
padding_text
=
chr
(
padding_times
)
*
padding_times
return
to_encrypt_content
+
padding_text
def
pkcs7unpadding_decrypted_content
(
self
,
decrypted_content
:
str
)
->
str
:
r
"""
处理使用PKCS7填充过的数据
:param decrypted_content
:return
"""
length
=
len
(
decrypted_content
)
unpadding
=
ord
(
decrypted_content
[
length
-
1
])
return
decrypted_content
[
0
:
length
-
unpadding
]
def
gen_aes_key_by_mysql_key
(
self
,
key
:
str
=
None
)
->
bytes
:
r
"""
生成mysql函数中使用的key实际对应的aes_key
:param key
"""
real_key
=
key
if
key
else
self
.
key
final_key
=
bytearray
(
16
)
for
i
,
c
in
enumerate
(
real_key
):
final_key
[
i
%
16
]
^=
ord
(
real_key
[
i
])
return
bytes
(
final_key
)
def
encrypt
(
self
,
to_encrypt_content
:
str
,
key
:
str
=
None
)
->
str
:
r
"""
mysql AES加密
模式 AES.MODE_ECB
填充pkcs7
加密后使用 base64 编码
:param to_encrypt_content
:param key
:return
"""
real_aes_key
=
self
.
gen_aes_key_by_mysql_key
(
key
)
if
key
else
self
.
real_aes_key
cipher
=
AES
.
new
(
real_aes_key
,
AES
.
MODE_ECB
)
# 将内容编码
to_encrypt_content
=
self
.
pkcs7padding_to_encrypt_content
(
to_encrypt_content
)
byte_content
=
to_encrypt_content
.
encode
(
'utf-8'
)
byte_encrypted
=
cipher
.
encrypt
(
byte_content
)
# base64 编码
byte_base64
=
base64
.
b64encode
(
byte_encrypted
)
return
str
(
byte_base64
,
'utf-8'
)
def
decrypt
(
self
,
to_decrypt_content
:
str
,
key
:
str
=
None
):
r
"""
mysql AES解密
模式 AES.MODE_ECB
去填充pkcs7
解密前使用 base64 解码
:param to_decrypt_content
:param key
:return
"""
real_aes_key
=
self
.
gen_aes_key_by_mysql_key
(
key
)
if
key
else
self
.
real_aes_key
cipher
=
AES
.
new
(
real_aes_key
,
AES
.
MODE_ECB
)
# 先将内容解码
byte_base64
=
to_decrypt_content
.
encode
(
'utf-8'
)
byte_encrypted
=
base64
.
b64decode
(
byte_base64
)
# 解码
result
=
cipher
.
decrypt
(
byte_encrypted
)
result
=
str
(
result
,
'utf-8'
)
# 去除填充内容
result
=
self
.
pkcs7unpadding_decrypted_content
(
result
)
return
result
cucc_common_pkg/ludq_utils/app_exception.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from
.app_response
import
AppResponse
from
.base_const
import
ConstResponseCode
from
..globalutility
import
Utility
class
AppException
(
Exception
):
"""
自定义的异常类, 表示可控的异常,不依赖任何其他应用基础类
"""
code
:
str
=
None
message
:
str
=
None
_status_code
:
int
=
None
def
__init__
(
self
,
code
:
str
=
None
,
message
:
str
=
None
,
status_code
:
int
=
200
):
r
"""
初始化异常
:param code
:param message
:param status_code
"""
self
.
code
:
str
=
code
or
ConstResponseCode
.
CODE_SYS_ERROR
self
.
message
:
str
=
message
or
ConstResponseCode
.
gen_msg_for_code
(
self
.
code
)
self
.
status_code
=
status_code
self
.
my_response
=
AppResponse
(
code
=
self
.
code
,
message
=
self
.
message
,
status_code
=
status_code
)
def
__str__
(
self
):
r
"""
字符串表示, 使用json字符串表示
Returns:
"""
return
str
(
self
.
my_response
)
def
__repr__
(
self
):
r
"""
字符串表示,使用 __str__ , 不会返回None,至少时 EMPTY字符串
Returns:
"""
return
repr
(
self
.
my_response
)
def
gen_err
(
self
)
->
dict
:
r
"""
生成带有 code,message的标准dict
Returns:
"""
return
self
.
my_response
.
gen_dict
()
@
property
def
status_code
(
self
)
->
int
:
r
"""
status_code 属性
:return:
"""
if
self
.
_status_code
is
None
or
self
.
_status_code
<
1
:
self
.
_status_code
=
200
return
self
.
_status_code
@
status_code
.
setter
def
status_code
(
self
,
status_code
:
int
):
r
"""
status_code 属性
:param status_code:
:return:
"""
self
.
_status_code
=
status_code
class
AppRuntimeException
(
Exception
):
"""
自定义的异常类, 区分其他的异常
还用于返回给前端基础的异常信息,和在后端打印的详细异常信息
"""
message
:
str
=
None
detail
:
str
=
None
code
:
str
=
None
def
__init__
(
self
,
message
:
str
,
detail
:
str
=
None
,
code
:
str
=
None
):
r
"""
初始化
:param message
:param detail
"""
self
.
message
:
str
=
message
self
.
detail
:
str
=
detail
self
.
code
:
str
=
code
def
__str__
(
self
):
r
"""
字符串表示
Returns:
"""
return
Utility
.
dict2jsonstr
({
"message"
:
self
.
message
,
"detail"
:
self
.
detail
,
"code"
:
self
.
code
})
def
__repr__
(
self
):
r
"""
字符串表示
Returns:
"""
return
self
.
__str__
()
cucc_common_pkg/ludq_utils/app_response.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from
.base_const
import
ConstResponseCode
from
..globalutility
import
Utility
class
AppResponse
:
r
"""
标准返回数据的封装,包含 code,msg,data等key的标准dict的json表示,不依赖任何其他应用基础类
"""
code
:
str
=
None
message
:
str
=
None
data
=
None
other_data
:
dict
=
None
_status_code
:
int
=
None
def
__init__
(
self
,
code
:
str
=
None
,
message
:
str
=
None
,
data
=
None
,
other_data
:
dict
=
None
,
status_code
:
int
=
200
,
):
r
"""
初始化,默认为OK的返回
:param code: 默认为 SysError
:param message: 默认为 系统错误
:param data:
:param other_data:
:param status_code:
"""
self
.
code
:
str
=
code
or
ConstResponseCode
.
CODE_OK
self
.
message
:
str
=
message
or
ConstResponseCode
.
gen_msg_for_code
(
self
.
code
)
self
.
other_data
=
other_data
if
other_data
is
not
None
else
dict
()
self
.
data
=
data
self
.
status_code
=
status_code
@
classmethod
def
from_dict
(
cls
,
source_dict
:
dict
,
key_for_msg
:
str
=
"message"
,
key_for_data
:
str
=
"data"
,
key_for_code
:
str
=
"code"
,
key_for_status_code
:
str
=
"status_code"
,
):
r
"""
根据 source_dict(包含 code 和 nessage key的dict) 生成 AppResponse
:param source_dict:
:param key_for_code:
:param key_for_data:
:param key_for_msg:
:param key_for_status_code:
:return:
"""
if
not
source_dict
:
raise
RuntimeError
(
"AppResponse解析dict对象出错:dict对象为空"
)
result
:
AppResponse
=
AppResponse
()
key_for_code
=
key_for_code
or
"code"
tmp_code
=
source_dict
.
get
(
key_for_code
)
if
not
tmp_code
:
raise
RuntimeError
(
Utility
.
join_str
(
"AppResponse解析dict对象出错:dict对象的"
,
key_for_code
,
"不合规"
))
result
.
code
=
str
(
tmp_code
)
key_for_msg
=
key_for_msg
or
"message"
tmp_msg
=
source_dict
.
get
(
key_for_msg
)
or
""
result
.
message
=
tmp_msg
key_for_data
=
key_for_data
or
"data"
tmp_data
=
source_dict
.
get
(
key_for_data
)
result
.
data
=
tmp_data
key_for_status_code
=
key_for_status_code
or
"status_code"
tmp_data
=
source_dict
.
get
(
key_for_status_code
)
result
.
status_code
=
tmp_data
for
tmp_key
,
tmp_value
in
source_dict
.
items
():
if
tmp_key
not
in
{
key_for_code
,
key_for_msg
,
key_for_data
,
key_for_status_code
}:
continue
result
.
other_data
[
tmp_key
]
=
tmp_value
return
result
def
__str__
(
self
):
r
"""
字符串表示, 使用json字符串表示
Returns:
"""
return
Utility
.
dict2jsonstr
(
self
.
gen_dict
())
def
__repr__
(
self
):
r
"""
字符串表示,使用 __str__ , 不会返回None,至少时 EMPTY字符串
Returns:
"""
return
self
.
__str__
()
def
gen_dict
(
self
)
->
dict
:
r
"""
生成带有 code,message的标准dict
Returns:
"""
ret_dict
=
{
"code"
:
self
.
code
,
"message"
:
self
.
message
,
"status_code"
:
self
.
status_code
}
if
self
.
other_data
:
for
key
,
value
in
self
.
other_data
.
items
():
ret_dict
[
key
]
=
value
if
self
.
data
is
not
None
:
ret_dict
[
'data'
]
=
self
.
data
return
ret_dict
@
property
def
status_code
(
self
)
->
int
:
r
"""
status_code 属性
:return:
"""
if
self
.
_status_code
is
None
or
self
.
_status_code
<
1
:
self
.
_status_code
=
200
return
self
.
_status_code
@
status_code
.
setter
def
status_code
(
self
,
status_code
:
int
):
r
"""
status_code 属性
:param status_code:
:return:
"""
self
.
_status_code
=
status_code
def
is_ok
(
self
)
->
bool
:
r
"""
验证code是否等于OK
"""
return
self
.
code
==
ConstResponseCode
.
CODE_OK
cucc_common_pkg/ludq_utils/base_const.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
class
ConstBaseApp
(
object
):
COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH
=
"/tmp/common_app_config.conf"
class
ConstResponseCode
(
object
):
CODE_OK
:
str
=
'OK'
CODE_SYS_ERROR
:
str
=
'SysError'
CODE_MISSING_PARAMETER
:
str
=
'MissingParameter'
CODE_STATUS_ERROR
:
str
=
'StatusError'
CODE_INVALID_ACTION
:
str
=
'InvalidAction'
CODE_AUTH_FAILURE
:
str
=
'AuthFailure'
CODE_UNAUTHORIZE_OPERATION
:
str
=
'UnauthorizedOperation'
CODE_K8S_ERROR
:
str
=
'K8sError'
CODE_MAP
:
dict
=
{
CODE_OK
:
"成功"
,
CODE_SYS_ERROR
:
"系统错误"
,
CODE_MISSING_PARAMETER
:
"参数错误"
,
CODE_STATUS_ERROR
:
"实例状态不允许当前的操作"
,
CODE_INVALID_ACTION
:
"请求的URI地址不存在"
,
CODE_AUTH_FAILURE
:
"登陆已超期"
,
CODE_UNAUTHORIZE_OPERATION
:
"API访问未授权"
,
CODE_K8S_ERROR
:
"K8S调用错误"
,
}
@
classmethod
def
gen_msg_for_code
(
cls
,
code
:
str
)
->
str
:
return
cls
.
CODE_MAP
.
get
(
code
,
code
)
cucc_common_pkg/ludq_utils/base_db_service.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
logging
from
typing
import
Optional
from
pymysql.connections
import
Connection
from
pymysql.cursors
import
DictCursor
from
.app_exception
import
AppRuntimeException
from
.common_app_config
import
CommonAppConfig
from
..globalutility
import
Utility
from
..my_stringutils
import
MyStringUtils
class
QueryResultWithFoundRows
:
result_list
:
Optional
[
list
]
=
None
found_rows
:
Optional
[
int
]
=
None
def
__init__
(
self
,
**
kwargs
):
self
.
result_list
=
kwargs
.
get
(
"result_list"
)
self
.
found_rows
=
kwargs
.
get
(
"found_rows"
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
class
DbBehaviourAfterExec
:
r
"""
数据库语句执行完成后的连接操作行为,不依赖任何其他应用基础类
默认 commit_after_exec 和 close_conn_after_exec 为 False
close_conn_when_exception 和 rollback_when_exception 为 True ,commit_when_exception 为 False
rollback_when_exception 的优先级比 commit_when_exception 高,
当 close_conn_when_exception 为True时,如果 rollback_when_exception 和 commit_when_exception 都为False,
则执行 rollback 操作
"""
commit_after_exec
:
bool
=
True
close_conn_after_exec
:
bool
=
True
close_conn_when_exception
:
bool
=
True
rollback_when_exception
:
bool
=
True
commit_when_exception
:
bool
=
False
def
__init__
(
self
,
commit_after_exec
:
bool
=
True
,
close_conn_after_exec
:
bool
=
True
,
close_conn_when_exception
:
bool
=
True
,
rollback_when_exception
:
bool
=
True
,
commit_when_exception
:
bool
=
False
):
r
"""
初始化数据库行为
:param commit_after_exec:
:param close_conn_after_exec:
:param close_conn_when_exception:
:param rollback_when_exception:
:param commit_when_exception:
"""
self
.
commit_after_exec
=
commit_after_exec
self
.
close_conn_after_exec
=
close_conn_after_exec
self
.
close_conn_when_exception
=
close_conn_when_exception
self
.
rollback_when_exception
=
rollback_when_exception
self
.
commit_when_exception
=
commit_when_exception
@
classmethod
def
gen_no_commit_close_behaviour
(
cls
):
r
"""
生成一个不commit,不close,异常时rollback、close的数据库行为
:return:
"""
return
DbBehaviourAfterExec
(
commit_after_exec
=
False
,
close_conn_after_exec
=
False
,
close_conn_when_exception
=
True
,
rollback_when_exception
=
True
,
commit_when_exception
=
False
)
class
BaseDbService
:
r
"""
基础 dbService类,定义一些基础函数,不依赖任何其他应用基础类
"""
def
__init__
(
self
,
logger
:
logging
.
Logger
=
None
):
self
.
logger
=
logger
or
CommonAppConfig
().
common_logger
def
join_str
(
self
,
*
to_join_list
:
str
,
separator_str
:
str
=
''
,
wrapper_str
:
str
=
None
,
ignore_none
:
bool
=
False
)
->
str
:
r
"""
连接 list, 当值为 None 时当做 null字符串 添加,但是不追加 wrapper_str
:param to_join_list
:param separator_str 默认使用空字符串
:param wrapper_str
:param ignore_none 默认为True
:return
"""
return
Utility
.
list_join_to_str
(
to_join_list
,
separator_str
=
separator_str
,
wrapper_str
=
wrapper_str
,
ignore_none
=
ignore_none
)
def
do_filter_mysql_param_with_single_quotes
(
self
,
to_filter_param
:
str
,
append_fix
:
str
=
"'"
)
->
str
:
r
"""
mysql防注入使用,用于过滤(替换)拼接字符串中的参数,当拼接`key`='value'时,对value中的特殊字符进行替换
:param to_filter_param
:param append_fix 默认为单引号
:return
"""
return
Utility
.
do_filter_mysql_param
(
to_filter_param
=
to_filter_param
,
append_fix
=
append_fix
)
def
do_filter_mysql_param_with_single_quotes_for_fuzzy_search
(
self
,
to_filter_param
:
str
,
append_fix
:
str
=
"'"
)
->
str
:
r
"""
mysql防注入使用,用于过滤(替换)拼接字符串中的参数,当拼接`key`='value'时,对value中的特殊字符进行替换
:param to_filter_param
:param append_fix 默认为单引号
:return
"""
return
Utility
.
do_filter_mysql_param_with_single_quotes_for_fuzzy_search
(
to_filter_param
=
to_filter_param
,
append_fix
=
append_fix
)
def
do_filter_mysql_param_list_with_single_quotes
(
self
,
to_filter_param_list
:
list
,
append_fix
:
str
=
"'"
)
->
Optional
[
list
]:
r
"""
mysql防注入使用,用于过滤(替换)拼接字符串中的参数,当拼接`key`='value'时,对value中的特殊字符进行替换
Utility.do_filter_mysql_param_list_with_single_quotes函数有bug,所以自己实现一下
:param to_filter_param_list
:param append_fix 默认为单引号
:return
"""
if
to_filter_param_list
is
None
:
return
None
return
[
self
.
do_filter_mysql_param_with_single_quotes
(
tmp_key
,
append_fix
=
append_fix
)
for
tmp_key
in
to_filter_param_list
]
def
gen_default_db_behaviour_after_exec
(
self
):
r
"""
生成默认的 db_behaviour_after_exec
默认行为为 正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:return:
"""
return
DbBehaviourAfterExec
(
commit_after_exec
=
True
,
close_conn_after_exec
=
True
,
close_conn_when_exception
=
True
,
rollback_when_exception
=
True
,
commit_when_exception
=
False
)
def
commit
(
self
,
conn
:
Connection
,
close_conn_after_exec
:
bool
=
True
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
bool
:
r
"""
执行commit操作
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param close_conn_after_exec
:param db_behaviour_after_exec
:return
"""
if
conn
is
None
:
return
True
conn
.
commit
()
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
if
close_conn_after_exec
or
(
db_behaviour_after_exec
and
db_behaviour_after_exec
.
close_conn_after_exec
):
conn
.
close
()
return
True
def
rollback
(
self
,
conn
:
Connection
,
close_conn_after_exec
:
bool
=
True
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
bool
:
r
"""
执行rollback操作
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param close_conn_after_exec
:param db_behaviour_after_exec
:return
"""
if
conn
is
None
:
return
True
conn
.
rollback
()
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
if
close_conn_after_exec
or
(
db_behaviour_after_exec
and
db_behaviour_after_exec
.
close_conn_after_exec
):
conn
.
close
()
return
True
def
close_conn
(
self
,
conn
:
Connection
):
r
"""
关闭连接
:param conn
"""
if
conn
is
None
:
return
conn
.
close
()
def
do_after_exec
(
self
,
conn
:
Connection
,
result
,
encountered_exception
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
):
r
"""
数据库执行完毕后的行为封装
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param result
:param encountered_exception
:param db_behaviour_after_exec
:return:
"""
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
if
not
encountered_exception
:
return
self
.
do_after_exec_without_exception
(
conn
,
result
,
db_behaviour_after_exec
)
else
:
return
self
.
do_after_exec_with_exception
(
conn
,
encountered_exception
,
db_behaviour_after_exec
)
def
do_after_exec_without_exception
(
self
,
conn
:
Connection
,
result
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
):
r
"""
数据库执行完毕后的行为封装
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param result
:param db_behaviour_after_exec
:return:
"""
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
close_conn_after_exec
=
True
if
db_behaviour_after_exec
.
close_conn_after_exec
else
False
commit_after_exec
=
True
if
db_behaviour_after_exec
and
db_behaviour_after_exec
.
commit_after_exec
else
False
if
commit_after_exec
and
conn
is
not
None
:
conn
.
commit
()
if
close_conn_after_exec
and
conn
is
not
None
:
conn
.
close
()
return
result
def
do_after_exec_with_exception
(
self
,
conn
:
Connection
,
encountered_exception
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
):
r
"""
数据库执行完毕后的行为封装
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param encountered_exception
:param db_behaviour_after_exec
:return:
"""
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
close_conn_when_exception
=
True
if
db_behaviour_after_exec
.
close_conn_when_exception
else
False
rollback_when_exception
=
True
if
db_behaviour_after_exec
.
rollback_when_exception
else
False
commit_when_exception
=
True
if
db_behaviour_after_exec
.
commit_when_exception
else
False
if
close_conn_when_exception
and
not
rollback_when_exception
and
not
commit_when_exception
:
# 当异常时关闭连接 但是 rollback_when_exception 和 commit_when_exception 都为False,则默认使用 rollback
rollback_when_exception
=
True
if
rollback_when_exception
and
conn
is
not
None
:
conn
.
rollback
()
elif
commit_when_exception
and
conn
is
not
None
:
conn
.
commit
()
if
close_conn_when_exception
and
conn
is
not
None
:
conn
.
close
()
raise
encountered_exception
def
querydb
(
self
,
conn
:
Connection
,
query_str
:
str
,
factory_method
=
None
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
list
:
r
"""
查询数据库
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param factory_method
:param db_behaviour_after_exec
:return
"""
if
conn
is
None
:
raise
AppRuntimeException
(
message
=
"未提供conn"
,
detail
=
"执行数据库操作时,传入的conn为None"
)
encountered_exception
:
Optional
[
BaseException
]
=
None
result
:
Optional
[
list
]
=
None
try
:
self
.
logger
.
debug
(
f
"执行sql语句:
{
query_str
}
"
)
with
conn
.
cursor
(
cursor
=
DictCursor
)
as
cursor
:
try
:
cursor
.
execute
(
query_str
)
result
=
cursor
.
fetchall
()
or
list
()
finally
:
cursor
.
close
()
self
.
logger
.
debug
(
f
"执行sql语句:
{
query_str
}
返回结果:
{
result
}
"
)
except
BaseException
as
e
:
encountered_exception
=
e
self
.
logger
.
debug
(
f
"执行sql语句:
{
query_str
}
发生异常:
{
e
}
"
)
if
not
encountered_exception
and
factory_method
is
not
None
and
result
:
try
:
result
=
[
factory_method
(
tmpDict
)
for
tmpDict
in
result
]
except
BaseException
as
e
:
encountered_exception
=
e
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
return
self
.
do_after_exec
(
conn
,
result
,
encountered_exception
,
db_behaviour_after_exec
)
def
querycount_db
(
self
,
conn
:
Connection
,
query_str
:
str
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
int
:
r
"""
查询数据库,执行count数据库查询,直接得到返回的整数
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param db_behaviour_after_exec
:return
"""
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
ret_list
=
self
.
querydb
(
conn
,
query_str
,
db_behaviour_after_exec
=
db_behaviour_after_exec
)
if
not
ret_list
:
raise
ValueError
(
'数据库查询语句返回结果集为空'
)
return
int
(
list
(
ret_list
[
0
].
values
())[
0
])
def
querydb_with_single_column_and_line
(
self
,
conn
:
Connection
,
query_str
:
str
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
Optional
[
str
]:
r
"""
查询数据库,执行返回单列单行的数据,并获取字符串,查询的数据不存在时,返回None
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param db_behaviour_after_exec
:return
"""
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
ret_list
=
self
.
querydb
(
conn
,
query_str
,
db_behaviour_after_exec
=
db_behaviour_after_exec
)
if
not
ret_list
:
return
None
return
MyStringUtils
.
to_str
(
list
(
ret_list
[
0
].
values
())[
0
])
def
querydb_with_sql_calc_found_rows
(
self
,
conn
:
Connection
,
query_str
:
str
,
factory_method
=
None
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
QueryResultWithFoundRows
:
r
"""
查询数据库,为带有 SQL_CALC_FOUND_ROWS 的sql语句返回 FOUND_ROWS()
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param query_str
:param factory_method
:param db_behaviour_after_exec
:return
"""
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
tmp_db_behaviour_after_exec
=
DbBehaviourAfterExec
.
gen_no_commit_close_behaviour
()
tmp_db_behaviour_after_exec
.
close_conn_when_exception
=
db_behaviour_after_exec
.
close_conn_when_exception
tmp_db_behaviour_after_exec
.
commit_when_exception
=
db_behaviour_after_exec
.
commit_when_exception
tmp_db_behaviour_after_exec
.
rollback_when_exception
=
db_behaviour_after_exec
.
rollback_when_exception
ret_list
=
self
.
querydb
(
conn
,
query_str
,
factory_method
=
factory_method
,
db_behaviour_after_exec
=
tmp_db_behaviour_after_exec
)
found_rows
=
self
.
querydb_with_single_column_and_line
(
conn
=
conn
,
query_str
=
"select FOUND_ROWS() as found_rows"
,
db_behaviour_after_exec
=
db_behaviour_after_exec
)
found_rows
=
int
(
found_rows
)
return
QueryResultWithFoundRows
(
result_list
=
ret_list
,
found_rows
=
found_rows
)
def
operate_db
(
self
,
conn
:
Connection
,
*
operate_sql_list
:
str
,
sql_list
:
list
=
None
,
db_behaviour_after_exec
:
DbBehaviourAfterExec
=
None
)
->
int
:
r
"""
操作数据库,返回受影响的行数, 可以有两种提供操作语句的方式,或者直接执行多条语句,或者使用 sql_list 提供操作语句列表
当 db_behaviour_after_exec为None时,使用默认的 db_behaviour_after_exec,即正常操作后commit、关闭数据库, 异常时rollback、关闭数据库
:param conn
:param operate_sql_list
:param sql_list
:param db_behaviour_after_exec
:return
"""
if
conn
is
None
:
raise
AppRuntimeException
(
message
=
"未提供conn"
,
detail
=
"执行数据库操作时,传入的conn为None"
)
encountered_exception
:
Optional
[
BaseException
]
=
None
result
:
int
=
0
try
:
# 先生成列表
real_sql_list
=
[]
if
operate_sql_list
:
real_sql_list
+=
operate_sql_list
elif
sql_list
:
real_sql_list
+=
sql_list
else
:
raise
ValueError
(
'未提供sql语句'
)
with
conn
.
cursor
(
cursor
=
DictCursor
)
as
cursor
:
try
:
for
tmp_sql_str
in
real_sql_list
:
self
.
logger
.
debug
(
f
"执行sql语句:
{
tmp_sql_str
}
"
)
result
+=
cursor
.
execute
(
tmp_sql_str
)
self
.
logger
.
debug
(
f
"执行sql语句:
{
tmp_sql_str
}
返回结果:
{
result
}
"
)
finally
:
cursor
.
close
()
except
BaseException
as
e
:
encountered_exception
=
e
if
db_behaviour_after_exec
is
None
:
db_behaviour_after_exec
=
self
.
gen_default_db_behaviour_after_exec
()
return
self
.
do_after_exec
(
conn
,
result
,
encountered_exception
,
db_behaviour_after_exec
)
cucc_common_pkg/ludq_utils/base_redis_service.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
random
from
typing
import
List
,
Dict
import
redis
from
redis
import
StrictRedis
from
redis.sentinel
import
Sentinel
from
.common_app_config
import
CommonAppConfig
from
.utils_base
import
UtilityBaseV2
class
BaseRedisService
:
r
"""
redis操作基类
"""
# 当调用jedis API时发生错误后重试的次数, 默认为1
retry_times_when_exception
:
int
=
1
app_utils
:
UtilityBaseV2
=
None
need_decode_responses
:
bool
=
True
r
"""
获取到redis函数结果后是否需要解码
如果conn的初始化中包含 decode_responses=True,则应该设置此属性为False,
即获取到的redis函数结果已经是解码后的数据了,不需要再次解码
"""
encoding_used_for_decode_response
:
str
=
"utf-8"
def
__init__
(
self
,
*
args
,
**
kwargs
):
r
"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key logger: 用于打印日志
"""
retry_times_when_exception
=
kwargs
.
get
(
"retry_times_when_exception"
)
app_utils
=
kwargs
.
get
(
"app_utils"
)
self
.
retry_times_when_exception
=
retry_times_when_exception
if
retry_times_when_exception
>
0
else
1
self
.
app_utils
=
app_utils
if
app_utils
else
UtilityBaseV2
(
logger
=
kwargs
.
get
(
"logger"
)
or
CommonAppConfig
().
common_logger
,
)
self
.
logger
=
kwargs
.
get
(
"logger"
)
or
CommonAppConfig
().
common_logger
def
set
(
self
,
name
:
str
,
value
:
str
,
ex
:
int
=
None
,
px
:
int
=
None
,
nx
:
bool
=
False
,
xx
:
bool
=
False
)
->
bool
:
r
"""
Set the value at key ``name`` to ``value`` , 返回 bool 表示是否成功执行
:param name
:param value
:param ex sets an expire flag on key ``name`` for ``ex`` seconds.
:param px sets an expire flag on key ``name`` for ``px`` milliseconds.
:param nx if set to True, set the value at key ``name`` to ``value`` only if it does not exist.
:param xx if set to True, set the value at key ``name`` to ``value`` only if it already exists.
"""
def
operation_func
():
return
self
.
get_master_conn
().
set
(
name
=
name
,
value
=
value
,
ex
=
ex
,
px
=
px
,
nx
=
nx
,
xx
=
xx
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
get
(
self
,
name
:
str
)
->
str
:
r
"""
Return the value at key ``name``, or None if the key doesn't exist,返回 utf-8解码的字符串
:param name
:return
"""
def
operation_func
():
result
=
self
.
get_slave_conn
().
get
(
name
=
name
)
if
self
.
need_decode_responses
and
result
is
not
None
:
result
=
str
(
result
,
self
.
encoding_used_for_decode_response
)
return
result
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
mset
(
self
,
name_value_dict
:
dict
)
->
bool
:
r
"""
Sets key/values based on a mapping. Mapping is a dictionary of
key/value pairs. Both keys and values should be strings or types that
can be cast to a string via str() ,
返回 bool 表示是否成功执行
:param name_value_dict
:return
"""
def
operation_func
():
return
self
.
get_master_conn
().
mset
(
mapping
=
name_value_dict
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
mget
(
self
,
*
to_get_names
:
str
)
->
list
:
r
"""
Returns a list of values ordered identically to ``keys``,
返回 utf-8解码的字符串list
:param to_get_names
:return
"""
def
operation_func
():
result
=
self
.
get_slave_conn
().
mget
(
keys
=
to_get_names
)
if
self
.
need_decode_responses
and
result
is
not
None
:
result
=
[
str
(
tmp_byte_value
,
self
.
encoding_used_for_decode_response
)
if
tmp_byte_value
is
not
None
else
None
for
tmp_byte_value
in
result
]
return
result
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
hset
(
self
,
name
:
str
,
key
:
str
,
value
:
str
)
->
int
:
r
"""
Set ``key`` to ``value`` within hash ``name``
Returns 1 if HSET created a new field, otherwise 0, 返回 int 表示是否成功插入的key的数量
:param name
:param key
:param value
"""
def
operation_func
():
return
self
.
get_master_conn
().
hset
(
name
=
name
,
key
=
key
,
value
=
value
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
hget
(
self
,
name
:
str
,
key
:
str
)
->
str
:
r
"""
Returns a list of values ordered identically to ``keys``,返回 utf-8解码的字符串list
:param name
:param key
"""
def
operation_func
():
result
=
self
.
get_slave_conn
().
hget
(
name
=
name
,
key
=
key
)
if
self
.
need_decode_responses
and
result
is
not
None
:
result
=
str
(
result
,
self
.
encoding_used_for_decode_response
)
return
result
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
hmset
(
self
,
name
:
str
,
key_value_dict
:
dict
)
->
bool
:
r
"""
Set key to value within hash ``name`` for each corresponding
key and value from the ``mapping`` dict. ,
返回 bool 表示是否成功执行
:param name
:param key_value_dict
"""
def
operation_func
():
return
self
.
get_master_conn
().
hmset
(
name
=
name
,
mapping
=
key_value_dict
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
hmget
(
self
,
*
to_get_keys
:
str
,
name
:
str
)
->
list
:
r
"""
Returns a list of values ordered identically to ``keys``,
返回 utf-8解码的字符串list
:param to_get_keys
:param name
"""
def
operation_func
():
result
=
self
.
get_slave_conn
().
hmget
(
name
=
name
,
keys
=
to_get_keys
)
if
self
.
need_decode_responses
and
result
is
not
None
:
result
=
[
str
(
tmp_byte_value
,
self
.
encoding_used_for_decode_response
)
if
tmp_byte_value
is
not
None
else
None
for
tmp_byte_value
in
result
]
return
result
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
hgetall
(
self
,
name
:
str
)
->
dict
:
r
"""
Return a Python dict of the hash's name/value pairs ,
返回 utf-8解码的key,value字符串 的 dict
:param name
"""
def
operation_func
():
result
=
self
.
get_slave_conn
().
hgetall
(
name
=
name
)
if
self
.
need_decode_responses
and
result
is
not
None
:
result
=
{
str
(
tmp_byte_key
,
self
.
encoding_used_for_decode_response
):
str
(
result
[
tmp_byte_key
],
self
.
encoding_used_for_decode_response
)
for
tmp_byte_key
in
result
.
keys
()}
return
result
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
expire
(
self
,
name
:
str
,
time
:
int
,
use_second
:
bool
=
False
,
use_millisecond
:
bool
=
False
)
->
bool
:
r
"""
设置key的过期时间, 从现在开始算过多少秒或毫秒之后key会被删除
如果 use_second 为 True ,则 time 单位为秒的int或者 python timedelta object.
如果 use_millisecond 为 True , 则 time 单位为毫秒的int或者 python timedelta object.
use_second 和 use_millisecond 必须至少有一个为True
返回 bool 表示是否成功执行
:param name
:param time
:param use_millisecond
:param use_second
"""
def
operation_func
():
if
use_second
:
return
self
.
get_master_conn
().
expire
(
name
=
name
,
time
=
time
)
elif
use_millisecond
:
return
self
.
get_master_conn
().
pexpire
(
name
=
name
,
time
=
time
)
else
:
raise
RuntimeError
(
'use_second 和 use_millisecond 必须至少有一个为True'
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
expireat
(
self
,
name
:
str
,
when
:
int
,
use_second
:
bool
=
False
,
use_millisecond
:
bool
=
False
)
->
bool
:
r
"""
设置key的过期时间, 在指定的时间点key会被删除
如果 use_second 为 True ,则 when 为 unix time or a Python datetime object
如果 use_millisecond 为 True , 则 when 为 unix time in milliseconds (unix time * 1000) or a Python datetime object.
use_second 和 use_millisecond 必须至少有一个为True
返回 bool 表示是否成功执行
:param name
:param when
:param use_second
:param use_millisecond
"""
def
operation_func
():
if
use_second
:
return
self
.
get_master_conn
().
expireat
(
name
=
name
,
when
=
when
)
elif
use_millisecond
:
return
self
.
get_master_conn
().
pexpireat
(
name
=
name
,
when
=
when
)
else
:
raise
RuntimeError
(
'use_second 和 use_millisecond 必须至少有一个为True'
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
delete
(
self
,
*
to_delete_names
:
str
)
->
int
:
r
"""
Delete one or more keys specified by ``names``
返回 int 表示是否成功删除的key的数量
:param to_delete_names
"""
def
operation_func
():
return
self
.
get_master_conn
().
delete
(
*
to_delete_names
)
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
keys
(
self
,
pattern
:
str
)
->
list
:
r
"""
Returns a list of keys matching ``pattern``,
返回 utf-8解码的字符串list
:param pattern
"""
def
operation_func
():
result
=
self
.
get_slave_conn
().
keys
(
pattern
=
pattern
)
if
self
.
need_decode_responses
and
result
is
not
None
:
result
=
[
str
(
tmp_byte_value
,
self
.
encoding_used_for_decode_response
)
for
tmp_byte_value
in
result
]
return
result
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
delete_by_pattern
(
self
,
pattern
:
str
)
->
int
:
r
"""
按 pattern 删除key,
返回 int 表示是否成功删除的key的数量
:param pattern
"""
def
operation_func
():
key_list
=
self
.
get_slave_conn
().
keys
(
pattern
=
pattern
)
if
key_list
:
return
self
.
get_master_conn
().
delete
(
*
key_list
)
else
:
return
0
return
self
.
app_utils
.
common_operation
(
retry_times_when_exception
=
self
.
retry_times_when_exception
,
operation_func
=
operation_func
,
on_operation_failed
=
self
.
on_operation_failed
,
on_operation_completed
=
self
.
on_operation_completed
)
def
on_operation_failed
(
self
):
r
"""
当redis操作失败时的动作,一般是重建连接,或者等待几秒之类的操作
:return:
"""
pass
def
on_operation_completed
(
self
):
r
"""
当redis操作完成时的动作,一般是什么都不做
:return:
"""
pass
def
get_master_conn
(
self
)
->
StrictRedis
:
r
"""
获取master连接
:return:
"""
raise
RuntimeError
(
"Not implemented"
)
def
get_slave_conn
(
self
)
->
StrictRedis
:
r
"""
获取slave连接
:return:
"""
raise
RuntimeError
(
"Not implemented"
)
class
RedisServiceMasterAndSlave
(
BaseRedisService
):
r
"""
master redis和slave redis分别设置的redis帮助服务
"""
master_redis_config_dict_list
:
List
[
Dict
]
=
None
slave_redis_config_dict_list
:
List
[
Dict
]
=
None
master_redis_conn_list
:
List
[
StrictRedis
]
=
None
slave_redis_conn_list
:
List
[
StrictRedis
]
=
None
def
__init__
(
self
,
*
args
,
**
kwargs
):
r
"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key master_redis_config_dict_list master redis连接的dict列表
:key slave_redis_config_dict_list slave redis连接的dict列表
字典的key和value组成参考 redis.Redis()的init函数,大致内容有
host='localhost', port=6379,
db=0, password=None, socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None, socket_keepalive_options=None,
connection_pool=None, unix_socket_path=None,
encoding='utf-8', encoding_errors='strict',
charset=None, errors=None,
decode_responses=False, retry_on_timeout=False,
ssl=False, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs='required', ssl_ca_certs=None,
ssl_check_hostname=False,
max_connections=None, single_connection_client=False,
health_check_interval=0, client_name=None, username=None
"""
super
().
__init__
(
*
args
,
**
kwargs
)
self
.
master_redis_config_dict_list
=
kwargs
.
get
(
"master_redis_config_dict_list"
)
self
.
slave_redis_config_dict_list
=
kwargs
.
get
(
"slave_redis_config_dict_list"
)
self
.
need_decode_responses
=
False
if
self
.
master_redis_config_dict_list
[
0
].
get
(
"decode_responses"
)
else
True
encoding_used_for_decode_response
=
self
.
master_redis_config_dict_list
[
0
].
get
(
"encoding"
)
self
.
encoding_used_for_decode_response
=
encoding_used_for_decode_response
if
encoding_used_for_decode_response
else
'utf-8'
self
.
_init_redis_conn
()
def
get_master_conn
(
self
):
return
self
.
master_redis_conn_list
[
random
.
choice
(
range
(
len
(
self
.
master_redis_conn_list
)))]
def
get_slave_conn
(
self
):
return
self
.
slave_redis_conn_list
[
random
.
choice
(
range
(
len
(
self
.
slave_redis_conn_list
)))]
def
on_operation_failed
(
self
):
self
.
_init_redis_conn
()
def
_init_redis_conn
(
self
):
self
.
master_redis_conn_list
=
[
redis
.
StrictRedis
(
**
tmp_dict
)
for
tmp_dict
in
self
.
master_redis_config_dict_list
]
self
.
slave_redis_conn_list
=
[
redis
.
StrictRedis
(
**
tmp_dict
)
for
tmp_dict
in
self
.
slave_redis_config_dict_list
]
class
RedisServiceSentinel
(
BaseRedisService
):
r
"""
哨兵版redis集群的redis帮助服务
"""
redis_sentinel_config_dict
:
dict
=
None
sentinel
:
Sentinel
=
None
service_name
:
str
=
None
def
__init__
(
self
,
*
args
,
**
kwargs
):
r
"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key service_name: 通过哨兵查询redis server使用的服务名
:key redis_sentinel_config_dict redis哨兵连接的dict配置信息
字典的key和value组成参考 redis.Sentinel 的init函数,大致内容有
sentinels 一组节点,每个节点使用 (hostname, port) 表示
min_other_sentinels 为一个哨兵定义一个peers的最小节点数,当查询一个哨兵时,如果没有达到这个阈值,其响应不被认为是有效的,默认为0
sentinel_kwargs 是一个字典,用于连接到哨兵时使用, 和 redis.Redis 的init函数中需要的参数一样,
如果没有提供该值,则使用 connection_kwargs 中以 socket_ 开头的参数作为 sentinel_kwargs
connection_kwargs 中的参数用于创建 redis server时使用
基本上 sentinel_kwargs 只需要 password=None, socket_timeout=None,
connection_kwargs 中需要 db=0, password=None, socket_timeout=None,
其中 sentinel_kwargs 和 connection_kwargs 可用的参数有
db=0, password=None, socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None, socket_keepalive_options=None,
connection_pool=None, unix_socket_path=None,
encoding='utf-8', encoding_errors='strict',
charset=None, errors=None,
decode_responses=False, retry_on_timeout=False,
ssl=False, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs='required', ssl_ca_certs=None,
ssl_check_hostname=False,
max_connections=None, single_connection_client=False,
health_check_interval=0, client_name=None, username=None
"""
super
().
__init__
(
*
args
,
**
kwargs
)
self
.
service_name
=
kwargs
.
get
(
"service_name"
)
self
.
redis_sentinel_config_dict
=
kwargs
.
get
(
"redis_sentinel_config_dict"
)
sentinels
=
self
.
redis_sentinel_config_dict
.
get
(
"sentinels"
)
min_other_sentinels
=
self
.
redis_sentinel_config_dict
.
get
(
"min_other_sentinels"
)
sentinel_kwargs
=
self
.
redis_sentinel_config_dict
.
get
(
"sentinel_kwargs"
)
connection_kwargs
=
self
.
redis_sentinel_config_dict
.
get
(
"connection_kwargs"
)
self
.
need_decode_responses
=
False
if
connection_kwargs
and
connection_kwargs
.
get
(
"decode_responses"
)
else
True
encoding_used_for_decode_response
=
connection_kwargs
.
get
(
"encoding"
)
if
connection_kwargs
else
None
self
.
encoding_used_for_decode_response
=
encoding_used_for_decode_response
if
encoding_used_for_decode_response
else
'utf-8'
if
connection_kwargs
:
self
.
sentinel
=
Sentinel
(
sentinels
=
sentinels
,
min_other_sentinels
=
min_other_sentinels
,
sentinel_kwargs
=
sentinel_kwargs
,
**
connection_kwargs
)
else
:
self
.
sentinel
=
Sentinel
(
sentinels
=
sentinels
,
min_other_sentinels
=
min_other_sentinels
,
sentinel_kwargs
=
sentinel_kwargs
)
def
get_master_conn
(
self
):
return
self
.
sentinel
.
master_for
(
service_name
=
self
.
service_name
)
def
get_slave_conn
(
self
):
return
self
.
sentinel
.
slave_for
(
service_name
=
self
.
service_name
)
def
on_operation_failed
(
self
):
pass
class
RedisServiceCluster
(
BaseRedisService
):
r
"""
集群版redis集群的redis帮助服务
"""
redis_server_config_dict_list
:
List
[
Dict
]
=
None
redis_server_conn_list
:
List
[
StrictRedis
]
=
None
def
__init__
(
self
,
*
args
,
**
kwargs
):
r
"""
初始化
:key retry_times_when_exception 当操作发生异常时的重试次数,默认为 1
:key app_utils: 使用的app_utils便捷类,默认自动生成
:key redis_server_config_dict_list redis连接的dict列表
字典的key和value组成参考 redis.Redis()的init函数,大致内容有
host='localhost', port=6379,
db=0, password=None, socket_timeout=None,
socket_connect_timeout=None,
socket_keepalive=None, socket_keepalive_options=None,
connection_pool=None, unix_socket_path=None,
encoding='utf-8', encoding_errors='strict',
charset=None, errors=None,
decode_responses=False, retry_on_timeout=False,
ssl=False, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs='required', ssl_ca_certs=None,
ssl_check_hostname=False,
max_connections=None, single_connection_client=False,
health_check_interval=0, client_name=None, username=None
"""
super
().
__init__
(
*
args
,
**
kwargs
)
self
.
redis_server_config_dict_list
=
kwargs
.
get
(
"redis_server_config_dict_list"
)
self
.
need_decode_responses
=
False
if
self
.
redis_server_config_dict_list
[
0
].
get
(
"decode_responses"
)
else
True
encoding_used_for_decode_response
=
self
.
redis_server_config_dict_list
[
0
].
get
(
"encoding"
)
self
.
encoding_used_for_decode_response
=
encoding_used_for_decode_response
if
encoding_used_for_decode_response
else
'utf-8'
self
.
_init_redis_conn
()
def
get_master_conn
(
self
):
return
self
.
redis_server_conn_list
[
random
.
choice
(
range
(
len
(
self
.
redis_server_conn_list
)))]
def
get_slave_conn
(
self
):
return
self
.
redis_server_conn_list
[
random
.
choice
(
range
(
len
(
self
.
redis_server_conn_list
)))]
def
on_operation_failed
(
self
):
self
.
_init_redis_conn
()
def
_init_redis_conn
(
self
):
self
.
redis_server_conn_list
=
[
redis
.
StrictRedis
(
**
tmp_dict
)
for
tmp_dict
in
self
.
redis_server_config_dict_list
]
cucc_common_pkg/ludq_utils/cert_utils.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
os
import
shutil
import
subprocess
import
tempfile
from
typing
import
Tuple
from
..globalutility
import
Utility
class
GenCertResult
:
root_key_content
:
str
=
None
root_key_filepath
:
str
=
None
root_cert_content
:
str
=
None
root_cert_filepath
:
str
=
None
ca_key_content
:
str
=
None
ca_key_filepath
:
str
=
None
ca_cert_content
:
str
=
None
ca_cert_filepath
:
str
=
None
cert_chain_content
:
str
=
None
def
__init__
(
self
,
**
kwargs
):
r
"""
:param kwargs:
:key root_key_content:
:key root_key_filepath:
:key root_cert_content:
:key root_cert_filepath:
:key ca_key_content:
:key ca_key_filepath:
:key ca_cert_content:
:key ca_cert_filepath:
:key cert_chain_content:
"""
self
.
root_key_content
=
kwargs
.
get
(
"root_key_content"
)
self
.
root_key_filepath
=
kwargs
.
get
(
"root_key_filepath"
)
self
.
root_cert_content
=
kwargs
.
get
(
"root_cert_content"
)
self
.
root_cert_filepath
=
kwargs
.
get
(
"root_cert_filepath"
)
self
.
ca_key_content
=
kwargs
.
get
(
"ca_key_content"
)
self
.
ca_key_filepath
=
kwargs
.
get
(
"ca_key_filepath"
)
self
.
ca_cert_content
=
kwargs
.
get
(
"ca_cert_content"
)
self
.
ca_cert_filepath
=
kwargs
.
get
(
"ca_cert_filepath"
)
self
.
cert_chain_content
=
kwargs
.
get
(
"cert_chain_content"
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
class
CertUtils
:
r
"""
证书相关的Utils类
"""
def
gen_root_key_and_cert_and_cluster_key_and_cert
(
self
,
keysize
:
int
=
4096
,
content_format
:
str
=
"pem"
,
**
kwargs
)
->
GenCertResult
:
r
"""
一次性生成根证书key和cert文件内容和生成中间证书key和cert文件内容
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:param kwargs:
:key rootca_org: 默认值 Istio
:key rootca_cn: 默认值 Root CA
:key rootca_days: 默认值 36500
:key intermediate_cluster_name: 必须提供,
:key dns_names: 默认值 [istiod.istio-system.svc]
:key ip_addresses: 默认值 []
:key intermediate_org: 默认值 Istio
:key intermediate_cn: 默认值 Intermediate CA
:key intermediate_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
dest_dir
=
kwargs
.
get
(
"dest_dir"
)
or
None
if
not
dest_dir
:
dest_dir
=
tempfile
.
mkdtemp
()
del_dir
=
kwargs
.
get
(
"del_dir"
)
if
del_dir
is
None
:
del_dir
=
True
kwargs
[
"dest_dir"
]
=
dest_dir
kwargs
[
"del_dir"
]
=
False
# 首先生成 root-key.pem 和 root-cert.pem
result1
=
self
.
gen_root_key_and_cert
(
keysize
=
keysize
,
content_format
=
content_format
,
**
kwargs
)
# 然后生成 ca-key.pem 和 ca-cert.pem
result2
=
self
.
gen_cluster_key_and_cert
(
keysize
=
keysize
,
content_format
=
content_format
,
root_key
=
result1
.
root_key_filepath
,
root_cert
=
result1
.
root_cert_filepath
,
**
kwargs
)
if
del_dir
:
shutil
.
rmtree
(
dest_dir
,
ignore_errors
=
True
)
return
result2
def
gen_root_key_and_cert
(
self
,
keysize
:
int
=
4096
,
content_format
:
str
=
"pem"
,
**
kwargs
)
->
GenCertResult
:
r
"""
一次性生成根证书key和cert文件内容
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:param kwargs:
:key rootca_org: 默认值 Istio
:key rootca_cn: 默认值 Root CA
:key rootca_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
dest_dir
=
kwargs
.
get
(
"dest_dir"
)
or
None
if
not
dest_dir
:
dest_dir
=
tempfile
.
mkdtemp
()
del_dir
=
kwargs
.
get
(
"del_dir"
)
if
del_dir
is
None
:
del_dir
=
True
kwargs
[
"dest_dir"
]
=
dest_dir
kwargs
[
"del_dir"
]
=
False
# 首先生成 root-key.pem
root_key_content
,
root_key_filepath
=
self
.
gen_key
(
keysize
=
keysize
,
content_format
=
content_format
,
dest_dir
=
dest_dir
,
del_dir
=
False
,
key_file_name
=
"root-key"
,
)
# 然后生成 root-cert.pem
root_cert_content
,
root_cert_filepath
=
self
.
gen_root_cert
(
root_key
=
root_key_filepath
,
keysize
=
keysize
,
content_format
=
content_format
,
**
kwargs
)
if
del_dir
:
shutil
.
rmtree
(
dest_dir
,
ignore_errors
=
True
)
return
GenCertResult
(
root_key_content
=
root_key_content
,
root_key_filepath
=
root_key_filepath
,
root_cert_content
=
root_cert_content
,
root_cert_filepath
=
root_cert_filepath
,
)
def
gen_cluster_key_and_cert
(
self
,
root_key
:
str
,
root_cert
:
str
,
keysize
:
int
=
4096
,
content_format
:
str
=
"pem"
,
**
kwargs
)
->
GenCertResult
:
r
"""
一次性生成中间证书key和cert文件内容
:param root_key: root_key文件内容或文件路径
:param root_cert: root_cert文件内容或文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:key intermediate_cluster_name: 必须提供,
:key dns_names: 默认值 [istiod.istio-system.svc]
:key ip_addresses: 默认值 []
:key intermediate_org: 默认值 Istio
:key intermediate_cn: 默认值 Intermediate CA
:key intermediate_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
dest_dir
=
kwargs
.
get
(
"dest_dir"
)
or
None
if
not
dest_dir
:
dest_dir
=
tempfile
.
mkdtemp
()
del_dir
=
kwargs
.
get
(
"del_dir"
)
if
del_dir
is
None
:
del_dir
=
True
kwargs
[
"dest_dir"
]
=
dest_dir
kwargs
[
"del_dir"
]
=
False
# 首先生成 ca-key.pem
ca_key_content
,
ca_key_filepath
=
self
.
gen_key
(
keysize
=
keysize
,
content_format
=
content_format
,
dest_dir
=
dest_dir
,
del_dir
=
False
,
key_file_name
=
"ca-key"
,
)
# 然后生成 ca-cert.pem
ca_cert_content
,
ca_cert_filepath
=
self
.
gen_intermediate_cert
(
root_key
=
root_key
,
root_cert
=
root_cert
,
ca_key
=
ca_key_filepath
,
keysize
=
keysize
,
content_format
=
content_format
,
**
kwargs
)
# 读取 root_key 内容
if
os
.
path
.
exists
(
root_key
)
and
os
.
path
.
isfile
(
root_key
):
root_key_filepath
=
root_key
with
open
(
root_key
,
'r'
,
encoding
=
'utf-8'
)
as
f
:
root_key_content
=
f
.
read
()
else
:
root_key_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-key.pem"
])
root_key_content
=
root_key
# 读取 root_cert 内容
if
os
.
path
.
exists
(
root_cert
)
and
os
.
path
.
isfile
(
root_cert
):
root_cert_filepath
=
root_cert
with
open
(
root_cert
,
'r'
,
encoding
=
'utf-8'
)
as
f
:
root_cert_content
=
f
.
read
()
else
:
root_cert_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-cert.pem"
])
root_cert_content
=
root_cert
if
del_dir
:
shutil
.
rmtree
(
dest_dir
,
ignore_errors
=
True
)
# 生成 cert_chain_content
cert_chain_content
=
f
"
{
ca_cert_content
}{
root_cert_content
}
"
return
GenCertResult
(
root_key_content
=
root_key_content
,
root_key_filepath
=
root_key_filepath
,
root_cert_content
=
root_cert_content
,
root_cert_filepath
=
root_cert_filepath
,
ca_key_content
=
ca_key_content
,
ca_key_filepath
=
ca_key_filepath
,
ca_cert_content
=
ca_cert_content
,
ca_cert_filepath
=
ca_cert_filepath
,
cert_chain_content
=
cert_chain_content
,
)
def
gen_key
(
self
,
keysize
:
int
=
4096
,
content_format
:
str
=
"pem"
,
**
kwargs
)
->
Tuple
[
str
,
str
]:
r
"""
生成key文件内容,返回 文件内容,文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:param kwargs:
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
:key key_file_name: 生成的key文件名,默认为 x-key
"""
supported_format
=
{
"pem"
}
if
content_format
not
in
supported_format
:
raise
ValueError
(
f
"不支持的格式,目前支持的格式:
{
supported_format
}
"
)
dest_dir
=
kwargs
.
get
(
"dest_dir"
)
or
None
if
not
dest_dir
:
dest_dir
=
tempfile
.
mkdtemp
()
del_dir
=
kwargs
.
get
(
"del_dir"
)
if
del_dir
is
None
:
del_dir
=
True
key_file_name
=
kwargs
.
get
(
"key_file_name"
)
or
"x-key"
x_key_filepath
=
os
.
sep
.
join
([
dest_dir
,
f
"
{
key_file_name
}
.pem"
])
# 使用命令 openssl genrsa -out root-key.pem ${ROOTCA_KEYSZ}
subprocess
.
run
(
f
"openssl genrsa -out
{
x_key_filepath
}
{
keysize
}
"
,
shell
=
True
,
check
=
True
,
timeout
=
10
)
# 读取文件内容获取结果
with
open
(
x_key_filepath
,
'r'
,
encoding
=
'utf-8'
)
as
f
:
result_content
=
f
.
read
()
# 尝试删除临时文件夹
if
del_dir
:
shutil
.
rmtree
(
dest_dir
,
ignore_errors
=
True
)
return
result_content
,
x_key_filepath
def
gen_root_cert
(
self
,
root_key
:
str
,
keysize
:
int
=
4096
,
content_format
:
str
=
"pem"
,
**
kwargs
)
->
Tuple
[
str
,
str
]:
r
"""
生成根证书文件内容,返回 文件内容,文件路径
:param root_key: root_key文件内容或文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:key rootca_org: 默认值 Istio
:key rootca_cn: 默认值 Root CA
:key rootca_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
supported_format
=
{
"pem"
}
if
content_format
not
in
supported_format
:
raise
ValueError
(
f
"不支持的格式,目前支持的格式:
{
supported_format
}
"
)
# 设置默认值
rootca_org
=
kwargs
.
get
(
"rootca_org"
)
or
"Istio"
rootca_cn
=
kwargs
.
get
(
"rootca_cn"
)
or
"Root CA"
rootca_days
=
kwargs
.
get
(
"rootca_days"
)
or
36500
dest_dir
=
kwargs
.
get
(
"dest_dir"
)
or
None
if
not
dest_dir
:
dest_dir
=
tempfile
.
mkdtemp
()
del_dir
=
kwargs
.
get
(
"del_dir"
)
if
del_dir
is
None
:
del_dir
=
True
if
os
.
path
.
exists
(
root_key
)
and
os
.
path
.
isfile
(
root_key
):
root_key_filepath
=
root_key
else
:
# 写入 root-key文件
root_key_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-key.pem"
])
with
open
(
root_key_filepath
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
f
.
write
(
root_key
)
# 生成 root-ca.conf
root_ca_conf_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-ca.conf"
])
with
open
(
root_ca_conf_filepath
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
f
.
write
(
f
"""[ req ]
encrypt_key = no
prompt = no
utf8 = yes
default_md = sha256
default_bits =
{
keysize
}
req_extensions = req_ext
x509_extensions = req_ext
distinguished_name = req_dn
[ req_ext ]
subjectKeyIdentifier = hash
basicConstraints = critical, CA:true
keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign
[ req_dn ]
O =
{
rootca_org
}
CN =
{
rootca_cn
}
"""
)
# 生成 root-cert.csr
root_cert_csr_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-cert.csr"
])
# 使用命令 openssl req -new -key root-key.pem -config root-ca.conf -out root-cert.csr
subprocess
.
run
(
f
"openssl req -new -key
{
root_key_filepath
}
-config
{
root_ca_conf_filepath
}
-out
{
root_cert_csr_filepath
}
"
,
shell
=
True
,
check
=
True
,
timeout
=
10
)
# 生成公钥 root-cert.pem
root_cert_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-cert.pem"
])
# 使用命令 openssl x509 -req -days ${ROOTCA_DAYS} -signkey root-key.pem -extensions req_ext -extfile root-ca.conf -in root-cert.csr -out root-cert.pem
subprocess
.
run
(
f
"openssl x509 -req -days
{
rootca_days
}
-signkey
{
root_key_filepath
}
-extensions req_ext -extfile
{
root_ca_conf_filepath
}
-in
{
root_cert_csr_filepath
}
-out
{
root_cert_filepath
}
"
,
shell
=
True
,
check
=
True
,
timeout
=
10
)
# 读取文件内容获取结果
with
open
(
root_cert_filepath
,
'r'
,
encoding
=
'utf-8'
)
as
f
:
result_content
=
f
.
read
()
# 尝试删除临时文件夹
if
del_dir
:
shutil
.
rmtree
(
dest_dir
,
ignore_errors
=
True
)
return
result_content
,
root_cert_filepath
def
gen_intermediate_cert
(
self
,
root_key
:
str
,
root_cert
:
str
,
ca_key
:
str
,
keysize
:
int
=
4096
,
content_format
:
str
=
"pem"
,
**
kwargs
)
->
Tuple
[
str
,
str
]:
r
"""
生成中间证书文件内容,返回 文件内容,文件路径
:param root_key: root_key文件内容或文件路径
:param root_cert: root_cert文件内容或文件路径
:param ca_key: ca_key文件内容或文件路径
:param keysize: 默认值 4096
:param content_format: 默认值 pem
:key dns_names: 默认值 [istiod.istio-system.svc]
:key ip_addresses: 默认值 []
:key intermediate_org: 默认值 Istio
:key intermediate_cn: 默认值 Intermediate CA
:key intermediate_cluster_name: 没有默认值
:key intermediate_days: 默认值 36500
:key dest_dir: 生成的文件存放目录, 不提供时使用临时目录,
:key del_dir: 生成文件完成后,是否删除文件夹, 默认值 True
"""
supported_format
=
{
"pem"
}
if
content_format
not
in
supported_format
:
raise
ValueError
(
f
"不支持的格式,目前支持的格式:
{
supported_format
}
"
)
# 设置默认值
dns_names
=
kwargs
.
get
(
"dns_names"
)
or
[
"istiod.istio-system.svc"
]
ip_addresses
=
kwargs
.
get
(
"ip_addresses"
)
or
None
intermediate_org
=
kwargs
.
get
(
"intermediate_org"
)
or
"Istio"
intermediate_cn
=
kwargs
.
get
(
"intermediate_cn"
)
or
"Intermediate CA"
intermediate_days
=
kwargs
.
get
(
"intermediate_days"
)
or
36500
intermediate_cluster_name
=
kwargs
.
get
(
"intermediate_cluster_name"
)
or
""
if
not
intermediate_cluster_name
:
raise
ValueError
(
f
"未提供intermediate_cluster_name"
)
dest_dir
=
kwargs
.
get
(
"dest_dir"
)
or
None
if
not
dest_dir
:
dest_dir
=
tempfile
.
mkdtemp
()
del_dir
=
kwargs
.
get
(
"del_dir"
)
if
del_dir
is
None
:
del_dir
=
True
# 写入 root-key文件
if
os
.
path
.
exists
(
root_key
)
and
os
.
path
.
isfile
(
root_key
):
root_key_filepath
=
root_key
else
:
# 写入 root-key文件
root_key_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-key.pem"
])
with
open
(
root_key_filepath
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
f
.
write
(
root_key
)
if
os
.
path
.
exists
(
root_cert
)
and
os
.
path
.
isfile
(
root_cert
):
root_cert_filepath
=
root_cert
else
:
# 写入 root-cert文件
root_cert_filepath
=
os
.
sep
.
join
([
dest_dir
,
"root-cert.pem"
])
with
open
(
root_cert_filepath
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
f
.
write
(
root_cert
)
if
os
.
path
.
exists
(
ca_key
)
and
os
.
path
.
isfile
(
ca_key
):
ca_key_filepath
=
ca_key
else
:
# 写入 ca_key 文件
ca_key_filepath
=
os
.
sep
.
join
([
dest_dir
,
"ca-key.pem"
])
with
open
(
ca_key_filepath
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
f
.
write
(
ca_key
)
# 生成 intermediate.conf
intermediate_conf_filepath
=
os
.
sep
.
join
([
dest_dir
,
"intermediate.conf"
])
# 生成 san 内容
dns_and_ip_list
=
list
()
if
dns_names
:
for
index
,
dns_name
in
enumerate
(
dns_names
):
dns_and_ip_list
.
append
(
f
"DNS.
{
index
}
=
{
dns_name
}
"
)
if
ip_addresses
:
for
index
,
ip_address
in
enumerate
(
ip_addresses
):
dns_and_ip_list
.
append
(
f
"IP.
{
index
}
=
{
ip_address
}
"
)
dns_and_ip_joined_str
=
"
\n
"
.
join
(
dns_and_ip_list
)
with
open
(
intermediate_conf_filepath
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
f
.
write
(
f
"""[ req ]
encrypt_key = no
prompt = no
utf8 = yes
default_md = sha256
default_bits =
{
keysize
}
req_extensions = req_ext
x509_extensions = req_ext
distinguished_name = req_dn
[ req_ext ]
subjectKeyIdentifier = hash
basicConstraints = critical, CA:true, pathlen:0
keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign
subjectAltName=@san
[ san ]
{
dns_and_ip_joined_str
}
[ req_dn ]
O =
{
intermediate_org
}
CN =
{
intermediate_cn
}
L =
{
intermediate_cluster_name
}
"""
)
# 生成 ca-cert.csr
ca_cert_csr_filepath
=
os
.
sep
.
join
([
dest_dir
,
"ca-cert.csr"
])
# 使用命令 openssl req -new -key ca-key.pem -config intermediate.conf -out ca-cert.csr
subprocess
.
run
(
f
"openssl req -new -key
{
ca_key_filepath
}
-config
{
intermediate_conf_filepath
}
-out
{
ca_cert_csr_filepath
}
"
,
shell
=
True
,
check
=
True
,
timeout
=
10
)
# 生成公钥 ca-cert.pem
ca_cert_filepath
=
os
.
sep
.
join
([
dest_dir
,
"ca-cert.pem"
])
# 使用命令 openssl x509 -req -days ${INTERMEDIATE_DAYS} -CA ../root-cert.pem -CAkey ../root-key.pem -CAcreateserial -extensions req_ext -extfile intermediate.conf -in ca-cert.csr -out ca-cert.pem
subprocess
.
run
(
f
"openssl x509 -req -days
{
intermediate_days
}
-CA
{
root_cert_filepath
}
-CAkey
{
root_key_filepath
}
-CAcreateserial -extensions req_ext -extfile
{
intermediate_conf_filepath
}
-in
{
ca_cert_csr_filepath
}
-out
{
ca_cert_filepath
}
"
,
shell
=
True
,
check
=
True
,
timeout
=
10
)
# 读取文件内容获取结果
with
open
(
ca_cert_filepath
,
'r'
,
encoding
=
'utf-8'
)
as
f
:
result_content
=
f
.
read
()
# 尝试删除临时文件夹
if
del_dir
:
shutil
.
rmtree
(
dest_dir
,
ignore_errors
=
True
)
return
result_content
,
ca_cert_filepath
cucc_common_pkg/ludq_utils/common_app_config.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
json
import
logging
import
os
import
traceback
from
typing
import
List
,
Dict
,
Union
import
yaml
from
.base_const
import
ConstBaseApp
class
CommonAppConfig
:
r
"""
应用的通用配置类,通过 CommonAppConfig() 获取
"""
# 通用配置
is_debug
:
bool
=
False
log_level_dict
=
{
"debug"
:
logging
.
DEBUG
,
"info"
:
logging
.
INFO
,
"warn"
:
logging
.
WARN
,
"warning"
:
logging
.
WARNING
,
"error"
:
logging
.
ERROR
,
"fatal"
:
logging
.
FATAL
,
"critical"
:
logging
.
CRITICAL
}
# 初始化字典中log_level使用字符串,初始化后转换为对应的log level int值
log_level
:
int
=
None
runtime_env
:
str
=
None
_common_logger
:
logging
.
Logger
=
None
r
"""通用的debug使用的logger"""
# 运行时内存缓存
runtime_cache_dict
:
dict
=
None
def
__new__
(
cls
,
*
args
,
**
kwargs
):
if
not
hasattr
(
cls
,
'inst'
):
# 单例
self
=
super
().
__new__
(
cls
,
*
args
,
**
kwargs
)
# 通过环境变量初始化,应用要连接的数据库、redis等其他配置
self
.
init_app_config
()
cls
.
inst
=
self
return
getattr
(
cls
,
'inst'
)
def
init_app_config
(
self
):
is_debug_str
=
os
.
environ
.
get
(
"IS_DEBUG"
)
or
"false"
self
.
is_debug
=
is_debug_str
.
lower
()
==
"true"
if
self
.
is_debug
:
self
.
log_level
=
logging
.
DEBUG
else
:
log_level_str
=
os
.
environ
.
get
(
"LOG_LEVEL"
)
or
"info"
log_level_int
=
self
.
log_level_dict
.
get
(
log_level_str
.
lower
())
if
log_level_int
is
None
:
log_level_int
=
logging
.
INFO
self
.
log_level
=
log_level_int
self
.
runtime_env
=
os
.
environ
.
get
(
"RUNTIME_ENV"
)
or
"NOTSET"
def
refresh_app_config
(
self
,
conf_filepath
:
str
=
ConstBaseApp
.
COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH
):
if
not
conf_filepath
:
conf_filepath
=
ConstBaseApp
.
COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH
try
:
app_config_dict
=
self
.
load_file_as_dict
(
conf_filepath
)
except
BaseException
as
e
:
if
conf_filepath
!=
ConstBaseApp
.
COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH
:
self
.
common_logger
.
warning
(
f
"读取配置文件
{
conf_filepath
}
时发生异常:
{
e
}
,不刷新CommonAppConfig"
)
self
.
common_logger
.
warning
(
traceback
.
format_exc
())
return
# 加载默认值
self
.
init_app_config
()
for
conf_key
in
{
"runtime_env"
}:
conf_value
=
app_config_dict
.
get
(
conf_key
)
if
conf_value
is
not
None
:
setattr
(
self
,
conf_key
,
conf_value
)
is_debug
=
app_config_dict
.
get
(
"is_debug"
)
if
is_debug
is
None
:
# 如果没有配置属性,则什么也不做
pass
else
:
if
isinstance
(
is_debug
,
bool
):
self
.
is_debug
=
is_debug
else
:
is_debug_str
=
str
(
is_debug
)
self
.
is_debug
=
is_debug_str
.
lower
()
==
"true"
if
self
.
is_debug
:
self
.
log_level
=
logging
.
DEBUG
else
:
log_level
=
app_config_dict
.
get
(
"log_level"
)
if
log_level
is
None
:
# 如果没有配置属性,则什么也不做
pass
else
:
if
isinstance
(
log_level
,
int
):
self
.
log_level
=
log_level
else
:
log_level_str
=
str
(
log_level
)
log_level_int
=
self
.
log_level_dict
.
get
(
log_level_str
.
lower
())
if
log_level_int
is
None
:
log_level_int
=
logging
.
INFO
self
.
log_level
=
log_level_int
return
def
load_file_as_dict
(
self
,
filepath
:
str
)
->
dict
:
r
"""
判断文件后缀名, 如果是 .yaml 或 .yml 则按 yaml 方式读取,
文件内容不合规,则抛出异常
"""
with
open
(
filepath
,
'r'
,
encoding
=
"utf-8"
)
as
dict_file
:
conf_content
=
dict_file
.
read
()
# 无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
dict_content
=
self
.
yaml_load
(
conf_content
)
if
isinstance
(
dict_content
,
str
):
raise
ValueError
(
f
"
{
filepath
}
的内容不是有效的json或yaml字符串"
)
return
dict_content
def
yaml_load
(
self
,
yaml_str
:
str
)
->
Union
[
dict
,
str
]:
r
"""
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
"""
dict_list
=
self
.
yaml_load_list
(
cfg_content
=
yaml_str
)
if
dict_list
:
return
dict_list
[
0
]
else
:
return
dict
()
def
yaml_load_list
(
self
,
path
:
str
=
None
,
cfg_content
:
str
=
None
)
->
List
[
Union
[
Dict
,
str
]]:
"""
将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据,每个dict放到list中返回,
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
:param path: 文件路径
:param cfg_content:
:return: list -> []
"""
if
path
:
cfg
=
self
.
read_file_content
(
file_path
=
path
)
else
:
cfg
=
cfg_content
or
""
yaml_generator
=
yaml
.
safe_load_all
(
cfg
)
# 将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据
if
yaml_generator
is
None
:
return
list
()
# 转成dict并保存到list中
yaml_list
=
list
()
for
one_yaml_generator
in
yaml_generator
:
if
one_yaml_generator
is
None
:
continue
yaml_list
.
append
(
json
.
loads
(
json
.
dumps
(
one_yaml_generator
)))
return
yaml_list
def
read_file_content
(
self
,
file_path
)
->
str
:
"""
读取文件内容
:param file_path: 文件路径
:return: str -> file-content
"""
with
open
(
file_path
,
'r'
)
as
f
:
# 打开一个文件,必须是'rb'模式打开
return
f
.
read
()
@
property
def
common_logger
(
self
):
r
"""
通用的debug使用的logger
"""
if
self
.
_common_logger
is
None
:
logger
=
logging
.
getLogger
(
"COMMON_LOGGER"
)
logger
.
setLevel
(
self
.
log_level
)
if
not
logger
.
handlers
:
ch
=
logging
.
StreamHandler
()
ch
.
setLevel
(
logging
.
DEBUG
)
formatter
=
logging
.
Formatter
(
'%(asctime)s %(name)s - %(pathname)s - func:%(funcName)s - lineno:%(lineno)s - %(levelname)s - %(message)s'
,
datefmt
=
'%Y-%m-%d %H:%M:%S'
)
ch
.
setFormatter
(
formatter
)
logger
.
addHandler
(
ch
)
self
.
_common_logger
=
logger
return
self
.
_common_logger
cucc_common_pkg/ludq_utils/common_base_webservice.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
datetime
import
logging
import
random
import
string
import
threading
import
traceback
from
typing
import
Union
,
Optional
from
flask
import
Request
,
make_response
,
Response
,
request
,
g
from
pymysql
import
Connection
from
.app_exception
import
AppException
,
AppRuntimeException
from
.app_response
import
AppResponse
from
.base_const
import
ConstResponseCode
,
ConstBaseApp
from
.base_db_service
import
DbBehaviourAfterExec
,
BaseDbService
from
.common_app_config
import
CommonAppConfig
from
.logined_user
import
LoginedUserInfo
from
.utils_http.http_cookie
import
HttpCookie
from
.utils_http.http_utils
import
HttpUtils
from
.utils_k8s.k8s_api_object_utils
import
OperationResultException
from
.utils_tg_configcenter.configcenter_utils
import
ConfigCenterUtils
from
.utils_tg_iam.iam_utils
import
IamUtils
from
.utils_tg_sso.sso_utils
import
SsoUtils
from
.utils_v2
import
UtilityV2
from
..my_stringutils
import
MyStringUtils
class
CommonBaseWebService
:
"""
基础微服务-基类,
继承类需要按照实际需要覆盖基类的 app_config() , get_app_product_code() , get_iam_domain() , get_config_center_domain() , app_utils() 方法,
还有 gen_app_db_conn() 方法, refresh_app_config() 等方法,
尤其是 refresh_app_config() 方法,如果需要动态更新应用logLevel和自己的应用配置,需要覆盖该方法
"""
need_login
:
bool
=
True
r
"""
应用是否需要登录,默认为True,继承类自行覆盖
"""
_conn
:
Connection
=
None
r
"""
应用使用的conn,因为conn使用完需要关闭,发生异常时可能无法关闭,所以在基类做统一的关闭处理,
如果具体的服务需要使用数据库连接,可以初始化这个属性,然后关闭操作可以交给基类
需要继承类实现 gen_app_db_conn 方法以初始化该属性, 示例:
def gen_app_db_conn(self) -> Connection:
raise AppRuntimeException(
message=f"TODO:gen_app_db_conn is To Implement",
detail=f"TODO:gen_app_db_conn is To Implement"
)
"""
no_commit_close_behaviour
=
DbBehaviourAfterExec
.
gen_no_commit_close_behaviour
()
commit_and_close_behaviour
=
DbBehaviourAfterExec
()
commit_and_no_close_behaviour
=
DbBehaviourAfterExec
(
commit_after_exec
=
True
,
close_conn_after_exec
=
False
,
close_conn_when_exception
=
True
,
rollback_when_exception
=
True
,
commit_when_exception
=
False
)
_app_product_code
:
str
=
None
r
"""
iam鉴权时使用的 product_code, 需要继承类实现 get_app_product_code 方法以初始化该属性, 示例:
def get_app_product_code(self) -> str:
raise AppRuntimeException(
message=f"TODO:get_app_product_code is To Implement",
detail=f"TODO:get_app_product_code is To Implement"
)
"""
# 应用使用的logger
logger
:
logging
.
Logger
=
None
_app_config
=
None
r
"""
app_config , 需要继承类自行实现该属性,示例:
@property
def app_config(self) -> MyAppConfig:
if self._app_config is None:
self._app_config = MyAppConfig.load_app_config()
return self._app_config
@classmethod
def load_app_config(cls, conf_filepath: str = None):
if not conf_filepath:
env_name = ConstGen.ENV_APP_CONFIG_FILEPATH
conf_filepath = os.environ.get(env_name)
if not conf_filepath:
conf_filepath = ConstGen.DEFAULT_APP_CONFIGFILE_PATH
app_config_dict = UtilityV2().load_file_as_dict(conf_filepath)
return MyAppConfig(**app_config_dict)
"""
common_app_config
:
CommonAppConfig
=
None
r
"""
应用的通用配置类
"""
is_debug
:
bool
=
False
# 应用使用的utils实例,减少utils创建次数
_app_utils
=
None
r
"""
应用使用的utils实例,减少utils创建次数,默认为 UtilityV2 类型,
如果应用有自己的继承自 UtilityV2 的util类,需要自行覆盖该属性,示例:
@property
def app_utils(self)->AppUtils:
if self._app_utils is None:
self._app_utils=AppUtils(logger=self.logger)
return self._app_utils
"""
http_utils
:
HttpUtils
=
None
r
"""
通用的http便捷类实例,减少utils创建次数
"""
request
:
Request
=
None
r
"""
应用处理的request
"""
request_json
:
dict
=
None
r
"""
API接收的json数据
"""
request_values
:
dict
=
None
r
"""
query参数和form-data或x-www-form-urlencoded传入的参数综合,即request.args和request.form的综合,key相同时query参数优先,先出现的key优先
"""
params_json
:
dict
=
None
r
"""
API接收的json数据,根据query参数或post body决定,两者都提供时, POST body 中的 json内容优先,不进行合并
"""
login_user
:
Optional
[
LoginedUserInfo
]
=
None
r
"""
访问服务的用户身份,通过 g.user_info 获取
"""
resp_cookie
:
HttpCookie
=
None
r
"""
用于设置 response Cookie , API 需要设置 Cookie时,设置该属性
"""
is_login
:
bool
=
False
r
"""
根据login_user是否存在,分析出的是否登录属性
"""
# 天宫个性化便捷类
_iam_domain
:
str
=
None
r
"""
生成iam_utils时使用的iam_domain属性, 需要继承类实现 get_iam_domain 方法以初始化该属性, 示例:
def get_iam_domain(self) -> str:
raise AppRuntimeException(
message=f"TODO:get_iam_domain is To Implement",
detail=f"TODO:get_iam_domain is To Implement"
)
"""
_iam_utils
:
IamUtils
=
None
_config_center_domain
:
str
=
None
r
"""
生成configcenter_utils时使用的config_center_domain属性, 需要继承类实现 get_config_center_domain 方法以初始化该属性, 示例:
def get_config_center_domain(self) -> str:
raise AppRuntimeException(
message=f"TODO:get_config_center_domain is To Implement",
detail=f"TODO:get_config_center_domain is To Implement"
)
"""
_configcenter_utils
:
ConfigCenterUtils
=
None
def
__init__
(
self
,
a_request
=
None
):
r
"""
初始化,并获取登陆用户信息以及是否登陆状态
:param a_request:
"""
# 初始化应用的config
self
.
common_app_config
=
CommonAppConfig
()
self
.
refresh_app_config
()
a_request
=
a_request
or
request
# 初始化应用使用的logger
random_str
=
''
.
join
([
random
.
choice
(
string
.
digits
+
string
.
ascii_letters
)
for
_
in
range
(
16
)])
self
.
logger
=
UtilityV2
.
gen_logger
(
logger_name
=
threading
.
currentThread
().
getName
()
+
"-"
+
random_str
,
log_level
=
CommonAppConfig
().
log_level
,
)
# 根据应用的config判断当前是否是debug模式
self
.
is_debug
=
self
.
common_app_config
.
is_debug
# 初始化通用的http便捷类
self
.
http_utils
:
HttpUtils
=
HttpUtils
(
incoming_request
=
a_request
,
logger
=
self
.
logger
)
# 初始化应用处理的request
self
.
request
=
a_request
# 初始化 self.request_values
self
.
request_values
=
self
.
request
.
values
.
to_dict
()
# 初始化 self.request_json
# 因为 self.request.json 在请求中的 Content - Type为application / json时才不为空,
# 而且此时body必须有内容,如果没有内容,调用 self.request.json 会报异常, 所以封装一个不会抛出异常的方法
self
.
request_json
=
self
.
gen_request_json_content
()
# 初始化 params_json
if
self
.
request_json
!=
{}:
self
.
params_json
=
self
.
request_json
else
:
params_in_query
=
self
.
request
.
args
.
get
(
'params'
)
if
params_in_query
:
self
.
params_json
=
self
.
app_utils
.
jsonstr2dict
(
params_in_query
)
if
self
.
params_json
is
None
:
self
.
params_json
=
dict
()
# 初始化应用的访问者身份
if
hasattr
(
g
,
"user_info"
)
and
isinstance
(
g
.
user_info
,
dict
):
self
.
login_user
:
LoginedUserInfo
=
SsoUtils
(
sso_api
=
"dontcare"
,
logger
=
self
.
logger
).
gen_user_info_from_sso_api_response
(
action
=
"从用户信息字典中获取用户信息"
,
user_info_dict
=
g
.
user_info
,
access_token
=
request
.
cookies
.
get
(
"accessToken"
),
)
else
:
self
.
login_user
:
Optional
[
LoginedUserInfo
]
=
None
self
.
is_login
=
self
.
login_user
is
not
None
and
self
.
login_user
.
is_login
()
def
do_service
(
self
):
r
"""
通用的服务执行模版,记录了服务访问时长以及捕获 SafeException 时返回 code, message json
请覆盖 my_do_service() 实现你自己的逻辑
:return:
"""
start_time
=
datetime
.
datetime
.
now
()
try
:
# 记录开始调用日志
self
.
log_start_call_info
()
# 获取服务返回结果
ret_obj
=
self
.
gen_ret_obj
()
# 计算服务时长,
end_time
=
datetime
.
datetime
.
now
()
consume_ms
=
(
end_time
-
start_time
).
total_seconds
()
*
1000
# 记录访问结束日志
self
.
log_end_call_info
(
consume_ms
=
consume_ms
,
ret_obj
=
ret_obj
)
# 尝试关闭服务连接
self
.
try_commit_and_close_conn
()
# 返回响应信息
resp
=
self
.
handle_ret_obj
(
ret_obj
)
if
self
.
resp_cookie
and
isinstance
(
resp
,
Response
):
# 设置cookie
self
.
resp_cookie
.
do_set_cookie
(
resp
)
if
isinstance
(
resp
,
Response
):
resp
.
headers
[
"Connection"
]
=
"close"
return
resp
except
AppException
as
safe_exception
:
self
.
try_commit_and_close_conn
()
return
self
.
handle_app_exception
(
safe_exception
,
start_time
)
except
BaseException
as
e
:
exception_tracback
=
traceback
.
format_exc
()
self
.
try_commit_and_close_conn
()
return
self
.
handle_common_exception
(
e
,
exception_tracback
,
start_time
)
def
log_start_call_info
(
self
):
r
"""
记录开始调用日志
"""
user_name
=
self
.
login_user
.
user_name
if
self
.
login_user
else
None
user_id
=
self
.
login_user
.
user_id
if
self
.
login_user
else
None
self
.
logger
.
info
(
self
.
app_utils
.
join_str_with_none
(
f
"
{
user_name
}
(
{
user_id
}
) 服务调用:"
,
self
.
__class__
.
__name__
,
" 开始处理时间:"
,
self
.
app_utils
.
now_daytime
(),
" values="
,
self
.
app_utils
.
dict2jsonstr
(
self
.
request_values
),
" json="
,
self
.
app_utils
.
dict2jsonstr
(
self
.
request_json
),
f
" 访问路径:
{
self
.
request
.
method
}
{
self
.
request
.
path
}
"
,
))
def
log_end_call_info
(
self
,
consume_ms
:
float
,
e
=
None
,
exception_tracback
=
None
,
ret_obj
=
None
):
r
"""
记录结束调用日志
"""
user_name
=
self
.
login_user
.
user_name
if
self
.
login_user
else
None
user_id
=
self
.
login_user
.
user_id
if
self
.
login_user
else
None
if
e
is
None
:
if
isinstance
(
ret_obj
,
(
AppResponse
,
str
)):
ret_str
=
str
(
ret_obj
)
elif
isinstance
(
ret_obj
,
dict
):
ret_str
=
self
.
app_utils
.
dict2jsonstr
(
ret_obj
)
elif
isinstance
(
ret_obj
,
list
):
ret_str
=
self
.
app_utils
.
list2jsonstr
(
ret_obj
)
else
:
ret_str
=
ret_obj
self
.
logger
.
info
(
self
.
app_utils
.
join_str_with_none
(
f
"
{
user_name
}
(
{
user_id
}
) 服务调用:"
,
self
.
__class__
.
__name__
,
f
" 正常结束,耗时
{
consume_ms
}
豪秒,返回内容:
{
ret_str
}
"
))
else
:
self
.
logger
.
error
(
self
.
app_utils
.
join_str_with_none
(
f
"
{
user_name
}
(
{
user_id
}
) 服务调用:"
,
self
.
__class__
.
__name__
,
f
" 异常结束,耗时
{
consume_ms
}
豪秒"
)
)
self
.
logger
.
error
(
self
.
join_str
(
self
.
__class__
.
__name__
,
f
'服务发生异常:
{
e
.
__class__
.
__name__
}
:
{
e
}
'
))
self
.
logger
.
error
(
exception_tracback
)
def
gen_ret_obj
(
self
):
r
"""
获取服务返回对象,
统一业务逻辑,减少业务代码
"""
# 统一业务逻辑,减少业务代码
if
self
.
need_login
and
not
self
.
is_login
:
ret_obj
=
self
.
action_when_not_login
()
else
:
# 执行业务逻辑
ret_obj
=
self
.
my_do_service
()
return
ret_obj
def
handle_ret_obj
(
self
,
ret_obj
):
r
"""
处理 ret_obj
"""
if
isinstance
(
ret_obj
,
AppResponse
):
resp
=
make_response
(
str
(
ret_obj
),
int
(
ret_obj
.
status_code
))
elif
isinstance
(
ret_obj
,
str
):
resp
=
make_response
(
ret_obj
)
elif
isinstance
(
ret_obj
,
dict
):
resp
=
make_response
(
self
.
app_utils
.
dict2jsonstr
(
ret_obj
))
elif
isinstance
(
ret_obj
,
list
):
resp
=
make_response
(
self
.
app_utils
.
list2jsonstr
(
ret_obj
))
else
:
resp
=
ret_obj
return
resp
def
my_do_service
(
self
):
r
"""
通用的 do_service(),调用该函数,请覆盖该函数实现你自己的逻辑
:return:
"""
raise
AppRuntimeException
(
message
=
f
"TODO:my_do_service is To Implement"
,
detail
=
f
"TODO:my_do_service is To Implement"
)
def
try_commit_and_close_conn
(
self
):
r
"""
尝试关闭conn
:return:
"""
try
:
BaseDbService
(
logger
=
self
.
logger
).
commit
(
conn
=
self
.
_conn
,
close_conn_after_exec
=
True
)
except
BaseException
:
pass
def
gen_ok_resp
(
self
,
data
=
None
):
r
"""
快速生成OK的返回信息
:param data:
:return:
"""
return
AppResponse
(
code
=
ConstResponseCode
.
CODE_OK
,
data
=
data
)
def
gen_sys_error_resp
(
self
,
data
=
None
):
r
"""
快速生成系统错误的返回信息
:param data:
:return:
"""
return
AppResponse
(
code
=
ConstResponseCode
.
CODE_SYS_ERROR
,
data
=
data
)
def
gen_not_login_err
(
self
)
->
AppResponse
:
r
"""
快速生成未登录错误
:return:
"""
return
AppResponse
(
code
=
ConstResponseCode
.
CODE_AUTH_FAILURE
)
def
raise_not_login_err
(
self
):
r
"""
快速抛出未登录异常
"""
raise
AppException
(
code
=
ConstResponseCode
.
CODE_AUTH_FAILURE
)
def
gen_no_auth_err
(
self
,
message
:
str
=
None
)
->
AppResponse
:
r
"""
快速生成缺少权限错误
:param message:
:return:
"""
return
AppResponse
(
code
=
ConstResponseCode
.
CODE_UNAUTHORIZE_OPERATION
,
message
=
message
)
def
raise_no_auth_err
(
self
,
message
:
str
=
None
):
r
"""
快速生成缺少权限错误
:param message:
:return:
"""
raise
AppException
(
code
=
ConstResponseCode
.
CODE_UNAUTHORIZE_OPERATION
,
message
=
message
)
def
gen_missing_param_err
(
self
,
message
:
str
=
None
):
r
"""
:param message:
:return:
"""
return
AppResponse
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
message
)
def
gen_status_err
(
self
,
message
:
str
=
None
):
r
"""
:param message:
:return:
"""
return
AppResponse
(
code
=
ConstResponseCode
.
CODE_STATUS_ERROR
,
message
=
message
)
def
raise_missing_param_err
(
self
,
message
:
str
=
None
):
r
"""
:param message:
:return:
"""
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
message
)
def
raise_status_err
(
self
,
message
:
str
=
None
):
r
"""
:param message:
:return:
"""
raise
AppException
(
code
=
ConstResponseCode
.
CODE_STATUS_ERROR
,
message
=
message
)
def
join_str
(
self
,
*
to_join_list
:
str
,
separator_str
:
str
=
''
,
wrapper_str
:
str
=
None
,
ignore_none
:
bool
=
False
)
->
str
:
r
"""
连接 list,当值为 None 时当做 null字符串添加,但是不追加wrapper_str
:param to_join_list
:param separator_str
:param wrapper_str
:param ignore_none
:return
"""
return
self
.
app_utils
.
list_join_to_str
(
to_join_list
,
separator_str
=
separator_str
,
wrapper_str
=
wrapper_str
,
ignore_none
=
ignore_none
,
)
def
resp_for_safe_exeception
(
self
,
safe_exception
:
AppException
):
r
"""
处理 AppException 时, 根据 AppException 返回的response
"""
return
make_response
(
self
.
app_utils
.
dict2jsonstr
(
safe_exception
.
gen_err
()),
int
(
safe_exception
.
status_code
))
def
handle_app_exception
(
self
,
safe_exception
:
AppException
,
start_time
):
r
"""
do_service函数中处理应用异常
:param start_time
:param safe_exception:
:return:
"""
# 计算服务时长,
end_time
=
datetime
.
datetime
.
now
()
consume_ms
=
(
end_time
-
start_time
).
total_seconds
()
*
1000
ret_obj
=
self
.
app_utils
.
dict2jsonstr
(
safe_exception
.
gen_err
())
# 记录访问结束日志
self
.
log_end_call_info
(
consume_ms
=
consume_ms
,
ret_obj
=
ret_obj
)
# 计算服务时长,如果超时则通知管理员
# 返回响应信息(错误信息)
resp
=
make_response
(
ret_obj
,
int
(
safe_exception
.
status_code
))
resp
.
headers
[
"Connection"
]
=
"close"
return
resp
def
resp_for_common_exeception
(
self
,
e
):
r
"""
处理 e 时, 根据 AppException 返回的response
"""
err_msg
=
f
"
{
e
.
__class__
.
__name__
}
:
{
e
}
"
code
=
ConstResponseCode
.
CODE_SYS_ERROR
if
isinstance
(
e
,
AppRuntimeException
):
self
.
logger
.
error
(
e
)
err_msg
=
e
.
message
code
=
e
.
code
or
ConstResponseCode
.
CODE_SYS_ERROR
if
isinstance
(
e
,
OperationResultException
):
self
.
logger
.
error
(
e
)
err_msg
=
e
.
failed_reason
code
=
e
.
code
or
ConstResponseCode
.
CODE_SYS_ERROR
response
=
AppResponse
(
code
=
code
,
message
=
err_msg
,
)
return
make_response
(
str
(
response
),
int
(
response
.
status_code
))
def
handle_common_exception
(
self
,
e
,
exception_tracback
,
start_time
):
r
"""
do_service函数中处理通用的异常
:param e:
:param exception_tracback:
:param start_time:
:return:
"""
# 计算服务时长,
end_time
=
datetime
.
datetime
.
now
()
consume_ms
=
(
end_time
-
start_time
).
total_seconds
()
*
1000
# 记录访问结束日志
self
.
log_end_call_info
(
consume_ms
=
consume_ms
,
e
=
e
,
exception_tracback
=
exception_tracback
)
# 通知管理员
resp
=
self
.
resp_for_common_exeception
(
e
)
resp
.
headers
[
"Connection"
]
=
"close"
return
resp
def
action_when_not_login
(
self
):
r
"""
do_service时如果未登录的行为,默认为需要登录则返回未登录异常
:return:
"""
return
self
.
gen_not_login_err
()
def
raise_uims_when_not_login
(
self
):
r
"""
生成带有uims返回地址的异常
:return:
"""
uims_api_for_frontpage
=
""
response
:
AppResponse
=
AppResponse
()
response
.
code
=
ConstResponseCode
.
CODE_AUTH_FAILURE
response
.
message
=
"请先登录uims并通过uims跳转到天宫后重试"
response
.
data
=
uims_api_for_frontpage
app_exception
=
AppException
()
app_exception
.
my_response
=
response
raise
app_exception
def
gen_request_json_content
(
self
):
r
"""
生成 request_json的内容
封装一个不会抛出异常的方法
:return:
"""
try
:
return
self
.
app_utils
.
jsonstr2dict
(
str
(
self
.
request
.
data
,
encoding
=
"utf-8"
))
or
dict
()
except
BaseException
as
e
:
# 打印一下错误信息
self
.
logger
.
error
(
"获取self.request.json时发生异常,使用空的dict,异常信息如下:"
)
self
.
logger
.
exception
(
e
,
exc_info
=
True
)
# 异常时返回空的dict
return
dict
()
def
gen_user_info
(
self
,
login_user
:
LoginedUserInfo
=
None
,
output_type
:
str
=
"str"
)
->
Union
[
str
,
dict
]:
r
"""
生成 {"user_id":"","user_name":""}格式的字典或字符串
:param login_user: 用户, 默认为 self.login_user
:param output_type: str or dict, 输出格式,默认为 str
"""
login_user
=
login_user
or
self
.
login_user
return
self
.
app_utils
.
gen_user_info
(
login_user
=
login_user
,
output_type
=
output_type
)
def
gen_app_db_conn
(
self
)
->
Connection
:
r
"""
:return:
"""
raise
AppRuntimeException
(
message
=
f
"TODO:gen_app_db_conn is To Implement"
,
detail
=
f
"TODO:gen_app_db_conn is To Implement"
)
@
property
def
conn
(
self
)
->
Connection
:
r
"""
conn 属性
:return:
"""
if
self
.
_conn
is
None
:
self
.
_conn
=
self
.
gen_app_db_conn
()
return
self
.
_conn
@
conn
.
setter
def
conn
(
self
,
conn
):
r
"""
conn 属性
:param conn:
:return:
"""
self
.
_conn
=
conn
def
get_iam_domain
(
self
)
->
str
:
r
"""
iam_domain 属性
"""
raise
AppRuntimeException
(
message
=
f
"TODO:get_iam_domain is To Implement"
,
detail
=
f
"TODO:get_iam_domain is To Implement"
)
@
property
def
iam_domain
(
self
)
->
str
:
r
"""
iam_domain 属性
:return:
"""
if
not
self
.
_iam_domain
:
self
.
_iam_domain
=
self
.
get_iam_domain
()
return
self
.
_iam_domain
@
iam_domain
.
setter
def
iam_domain
(
self
,
iam_domain
):
r
"""
iam_domain 属性
:param iam_domain:
:return:
"""
self
.
_iam_domain
=
iam_domain
@
property
def
iam_utils
(
self
):
r
"""
iam_utils 属性
:return:
"""
if
self
.
_iam_utils
is
None
:
self
.
_iam_utils
:
IamUtils
=
IamUtils
(
iam_api
=
self
.
iam_domain
+
"/iam"
,
access_token
=
self
.
login_user
.
access_token
if
self
.
login_user
else
MyStringUtils
.
EMPTY
,
http_utils
=
self
.
http_utils
,
app_utils
=
self
.
app_utils
)
return
self
.
_iam_utils
def
get_config_center_domain
(
self
)
->
str
:
r
"""
config_center_domain 属性
"""
raise
AppRuntimeException
(
message
=
f
"TODO:get_config_center_domain is To Implement"
,
detail
=
f
"TODO:get_config_center_domain is To Implement"
)
@
property
def
config_center_domain
(
self
)
->
str
:
r
"""
config_center_domain 属性
:return:
"""
if
not
self
.
_config_center_domain
:
self
.
_config_center_domain
=
self
.
get_config_center_domain
()
return
self
.
_config_center_domain
@
config_center_domain
.
setter
def
config_center_domain
(
self
,
config_center_domain
):
r
"""
config_center_domain 属性
:param config_center_domain:
:return:
"""
self
.
_config_center_domain
=
config_center_domain
@
property
def
configcenter_utils
(
self
)
->
ConfigCenterUtils
:
r
"""
configcenter_utils 属性
:return:
"""
if
self
.
_configcenter_utils
is
None
:
self
.
_configcenter_utils
:
ConfigCenterUtils
=
ConfigCenterUtils
(
configcenter_api
=
self
.
config_center_domain
+
"/configcenter"
,
access_token
=
self
.
login_user
.
access_token
if
self
.
login_user
else
MyStringUtils
.
EMPTY
,
http_utils
=
self
.
http_utils
,
app_utils
=
self
.
app_utils
)
return
self
.
_configcenter_utils
@
property
def
app_utils
(
self
)
->
UtilityV2
:
r
"""
app_utils 属性
:return:
"""
if
self
.
_app_utils
is
None
:
self
.
_app_utils
=
UtilityV2
(
logger
=
self
.
logger
)
return
self
.
_app_utils
@
app_utils
.
setter
def
app_utils
(
self
,
app_utils
):
r
"""
app_utils 属性
:param app_utils:
:return:
"""
self
.
_app_utils
=
app_utils
@
property
def
app_config
(
self
):
r
"""
app_config 属性
:return:
"""
if
self
.
_app_config
is
None
:
raise
AppRuntimeException
(
message
=
f
"TODO:app_config is To Implement"
,
detail
=
f
"TODO:app_config is To Implement"
)
return
self
.
_app_config
@
app_config
.
setter
def
app_config
(
self
,
app_config
):
r
"""
app_config 属性
:param app_config:
:return:
"""
self
.
_app_config
=
app_config
def
get_app_product_code
(
self
)
->
str
:
r
"""
app_product_code 属性
"""
raise
AppRuntimeException
(
message
=
f
"TODO:get_app_product_code is To Implement"
,
detail
=
f
"TODO:get_app_product_code is To Implement"
)
@
property
def
app_product_code
(
self
)
->
str
:
r
"""
app_product_code 属性
:return:
"""
if
self
.
_app_product_code
is
None
:
self
.
_app_product_code
=
self
.
get_app_product_code
()
return
self
.
_app_product_code
@
app_product_code
.
setter
def
app_product_code
(
self
,
app_product_code
):
r
"""
app_product_code 属性
:param app_product_code:
:return:
"""
self
.
_app_product_code
=
app_product_code
def
refresh_app_config
(
self
):
r
"""
刷新 common_app_config 配置 和 应用配置,
默认为每次都读取 ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH 文件刷新 common_app_config ,
如果继承类需要每次都重新抓取应用配置,则需要覆盖该方法 示例:
def refresh_app_config(self):
hot_reload_from_env = os.environ.get('HOT_RELOAD_CONFIG')
hot_reload = hot_reload_from_env == "true"
if hot_reload:
# 使用应用提供的配置文件刷新 common_app_config
self.common_app_config.refresh_app_config(conf_filepath=MyAppConfig.get_app_conf_filepath())
# 使用应用提供的配置文件刷新 app_config
self.app_config.refresh_app_config(conf_filepath=MyAppConfig.get_app_conf_filepath())
或者
def refresh_app_config(self):
# 仍然使用 ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH 刷新 common_app_config
super().refresh_app_config()
hot_reload_from_env = os.environ.get('HOT_RELOAD_CONFIG')
hot_reload = hot_reload_from_env == "true"
if hot_reload:
# 使用应用提供的配置文件刷新 app_config
self.app_config.refresh_app_config(conf_filepath=MyAppConfig.get_app_conf_filepath())
"""
# 使用 ConstBaseApp.COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH 刷新 common_app_config
self
.
common_app_config
.
refresh_app_config
(
ConstBaseApp
.
COMMON_APP_CONFIG_DEFAULT_CONF_FILE_PATH
)
cucc_common_pkg/ludq_utils/common_base_webservice_for_stream.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from
flask
import
make_response
,
Response
from
.app_exception
import
AppRuntimeException
,
AppException
from
.app_response
import
AppResponse
from
.common_base_webservice
import
CommonBaseWebService
from
.utils_k8s.k8s_api_object_utils
import
OperationResultException
class
CommonBaseWebServiceForStream
(
CommonBaseWebService
):
"""
基础微服务-基类(文件流类型的微服务基础),依赖一些其他应用基础类
"""
def
action_when_not_login
(
self
):
return
make_response
(
"未登录"
,
400
)
def
handle_ret_obj
(
self
,
ret_obj
):
r
"""
处理 ret_obj
"""
# 如果返回 AppResponse,必然是CODE不为OK的Response
if
isinstance
(
ret_obj
,
AppResponse
):
resp
=
make_response
(
ret_obj
.
message
,
400
)
elif
isinstance
(
ret_obj
,
Response
):
resp
=
ret_obj
else
:
raise
AppRuntimeException
(
message
=
"服务内部错误:处理结果类型错误"
,
detail
=
f
"ret_obj类型=
{
type
(
ret_obj
)
}
, ret_obj=
{
ret_obj
}
"
)
return
resp
def
resp_for_safe_exeception
(
self
,
safe_exception
:
AppException
):
r
"""
处理 AppException 时, 根据 AppException 返回的response
"""
return
make_response
(
safe_exception
.
message
,
400
)
def
resp_for_common_exeception
(
self
,
e
):
r
"""
处理 e 时, 根据 AppException 返回的response
"""
err_msg
=
f
"
{
e
.
__class__
.
__name__
}
:
{
e
}
"
if
isinstance
(
e
,
AppRuntimeException
):
self
.
logger
.
error
(
e
)
err_msg
=
e
.
message
if
isinstance
(
e
,
OperationResultException
):
self
.
logger
.
error
(
e
)
err_msg
=
e
.
failed_reason
return
make_response
(
f
"服务发生异常:
{
err_msg
}
"
,
500
)
cucc_common_pkg/ludq_utils/logined_user.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
from
.app_exception
import
AppRuntimeException
from
..globalutility
import
Utility
class
LoginedUserInfo
:
r
"""
登陆用户的信息POJO类,不依赖任何其他应用基础类
"""
def
__init__
(
self
):
self
.
account_id
=
None
self
.
account_name
=
None
self
.
user_id
=
None
self
.
user_name
=
None
self
.
mobile
=
None
self
.
email
=
None
self
.
is_root
=
None
self
.
session_uuid
=
None
r
"""
目前获取不到session_uuid
"""
self
.
access_token
=
None
r
"""
继续调用其他API时使用的access_token
"""
def
is_login
(
self
):
return
not
(
self
.
user_id
is
None
or
self
.
user_id
==
''
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
@
classmethod
def
from_dict
(
cls
,
user_info_dict
,
access_token
:
str
=
None
):
r
"""
根据sso_api返回的用户信息dict转换成LoginedUserInfo
:param user_info_dict
:param access_token:
"""
return
cls
().
fill_attr_by_dict
(
user_info_dict
,
access_token
)
def
fill_attr_by_dict
(
self
,
user_info_dict
:
dict
,
access_token
:
str
=
None
):
r
"""
根据sso_api返回的用户信息dict填充自身属性,返回自身
:param user_info_dict:
:param access_token:
:return:
"""
tmp_value
=
user_info_dict
.
get
(
'accountID'
)
if
not
tmp_value
:
raise
AppRuntimeException
(
message
=
f
"通过字典转换成LoginedUserInfo对象失败"
,
detail
=
f
"字典内容不包括accountID,user_info_dict=
{
Utility
.
dict2jsonstr
(
user_info_dict
)
}
"
)
self
.
account_id
=
tmp_value
tmp_value
=
user_info_dict
.
get
(
'userID'
)
if
not
tmp_value
:
raise
AppRuntimeException
(
message
=
f
"通过字典转换成LoginedUserInfo对象失败"
,
detail
=
f
"字典内容不包括userID,user_info_dict=
{
Utility
.
dict2jsonstr
(
user_info_dict
)
}
"
)
self
.
user_id
=
tmp_value
tmp_value
=
user_info_dict
.
get
(
'userName'
)
if
not
tmp_value
:
raise
AppRuntimeException
(
message
=
f
"通过字典转换成LoginedUserInfo对象失败"
,
detail
=
f
"字典内容不包括userName,user_info_dict=
{
Utility
.
dict2jsonstr
(
user_info_dict
)
}
"
)
self
.
user_name
=
tmp_value
self
.
account_name
=
user_info_dict
.
get
(
'accountName'
)
self
.
mobile
=
user_info_dict
.
get
(
'mobile'
)
self
.
email
=
user_info_dict
.
get
(
'email'
)
self
.
is_root
=
user_info_dict
.
get
(
'isRoot'
)
self
.
access_token
=
access_token
return
self
cucc_common_pkg/ludq_utils/md5_utils.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
hashlib
class
Md5Utils
:
r
"""
md5 utils类,不依赖任何其他应用基础类
"""
def
md5_hash
(
self
,
file_path
,
read_byte_once
=
1024
):
r
"""
获取文件的md5哈希值
"""
md5_1
=
hashlib
.
md5
()
# 创建一个md5算法对象
with
open
(
file_path
,
'rb'
)
as
f
:
# 打开一个文件,必须是'rb'模式打开
while
1
:
data
=
f
.
read
(
read_byte_once
)
# 由于是一个文件,每次只读取固定字节
if
data
:
# 当读取内容不为空时对读取内容进行update
md5_1
.
update
(
data
)
else
:
# 当整个文件读完之后停止update
break
ret
=
md5_1
.
hexdigest
()
# 获取这个文件的MD5值
return
ret
cucc_common_pkg/ludq_utils/rsa_utils.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
base64
import
rsa
class
CannotDecryptContentException
(
Exception
):
"""
自定义的异常类, 用于表示不能将加密后的内容进行界面的异常
"""
def
__init__
(
self
,
message
:
str
):
r
"""
初始化
:param message
"""
self
.
message
:
str
=
message
def
__str__
(
self
):
r
"""
字符串表示
Returns:
"""
return
self
.
message
def
__repr__
(
self
):
r
"""
字符串表示
Returns:
"""
return
self
.
message
class
RsaUtils
:
r
"""
RSA utils类,不依赖任何其他应用基础类
"""
def
__init__
(
self
,
key
:
str
=
None
):
r
"""
初始化
:param key: pkcs1格式,即 -----BEGIN RSA PRIVATE KEY-----
"""
self
.
key
:
str
=
key
def
encrypt
(
self
,
to_encrypt_content
:
str
,
public_key
:
str
):
r
"""
RSA加密
:param to_encrypt_content: 待加密的字符串
:param public_key: 公钥,pkcs1格式,即 -----BEGIN RSA PUBLIC KEY-----
:return: 加密后的字符串
"""
to_encrypt_content
=
to_encrypt_content
.
encode
(
"utf-8"
)
rsa_public_key
=
rsa
.
PublicKey
.
load_pkcs1
(
public_key
.
encode
(
"utf-8"
),
format
=
'PEM'
)
return
base64
.
b64encode
(
rsa
.
encrypt
(
to_encrypt_content
,
rsa_public_key
)).
decode
(
"utf-8"
)
def
gen_rsa_key
(
self
)
->
tuple
:
r
"""
生成公钥私钥
:return:
"""
return
rsa
.
newkeys
(
512
)
def
decrypt
(
self
,
to_decrypt_content
:
str
,
key
:
str
=
None
)
->
str
:
r
"""
RSA解密
:param to_decrypt_content
:param key:pkcs1格式,即 -----BEGIN RSA PRIVATE KEY-----
"""
real_key
=
key
if
key
else
self
.
key
private_key
=
rsa
.
PrivateKey
.
load_pkcs1
(
real_key
.
encode
(
"utf-8"
),
format
=
'PEM'
)
try
:
content
=
rsa
.
decrypt
(
base64
.
b64decode
(
to_decrypt_content
),
private_key
)
return
content
.
decode
(
"utf-8"
)
except
BaseException
as
e
:
encountered_e
=
CannotDecryptContentException
(
str
(
e
))
if
encountered_e
:
raise
encountered_e
cucc_common_pkg/ludq_utils/send_notices_utils.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
logging
import
smtplib
from
email.header
import
Header
from
email.message
import
Message
from
email.mime.text
import
MIMEText
from
typing
import
Union
import
requests
from
..globalutility
import
Utility
from
..my_stringutils
import
MyStringUtils
class
ToReceiveNoticeUser
:
r
"""
接收通知人员POJO类,不依赖任何其他应用基础类
"""
phone
:
str
=
None
email
:
str
=
None
def
__init__
(
self
,
source
:
dict
=
None
,
phone
:
str
=
None
,
email
:
str
=
None
):
r
"""
根据 source(包含 phone 和 email 初始化 ToReceiveNoticeUser
:param source:
:param phone:
:param email:
"""
if
source
is
None
:
source
=
dict
()
self
.
phone
=
phone
or
source
.
get
(
"phone"
)
self
.
email
=
email
or
source
.
get
(
"email"
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
class
NoticeSendResult
:
r
"""
发送通知结果POJO类,不依赖任何其他应用基础类
"""
email_send_result
:
bool
=
None
sms_send_result
:
bool
=
None
def
__init__
(
self
):
r
"""
初始化
"""
self
.
email_send_result
=
False
self
.
sms_send_result
=
False
def
is_all_success
(
self
):
r
"""
是否全部发送成功便捷方法
"""
return
self
.
email_send_result
and
self
.
sms_send_result
def
is_any_success
(
self
):
r
"""
是否任意发送成功便捷方法
"""
return
self
.
email_send_result
or
self
.
sms_send_result
class
SendEmailConfig
:
r
"""
发送邮件配置类,不依赖任何其他应用基础类
"""
def
__init__
(
self
,
source
:
dict
=
None
,
mail_host
:
str
=
None
,
mail_host_port
:
str
=
None
,
mail_from_account
:
str
=
None
,
mail_from_account_dwp
:
str
=
None
):
r
"""
根据 source(包含 mail_host 和 mail_from_account 和 mail_from_account_dwp key的dict) 初始化 SendEmailConfig
:param source:
:param mail_host:
:param mail_host_port:
:param mail_from_account:
:param mail_from_account_dwp:
"""
if
source
is
None
:
source
=
dict
()
self
.
mail_host
:
str
=
mail_host
or
source
.
get
(
"mail_host"
)
self
.
mail_host_port
:
str
=
mail_host_port
or
source
.
get
(
"mail_host_port"
)
or
25
self
.
mail_from_account
:
str
=
mail_from_account
or
source
.
get
(
"mail_from_account"
)
self
.
mail_from_account_dwp
:
str
=
mail_from_account_dwp
or
source
.
get
(
"mail_from_account_dwp"
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
class
SendSmsConfig
:
r
"""
发送短信配置类,不依赖任何其他应用基础类
"""
def
__init__
(
self
,
source
:
dict
=
None
,
api_server
:
str
=
None
,
sys_name
:
str
=
None
,
sys_token
:
str
=
None
):
r
"""
根据 source(包含 api_server 和 sys_name 和 sys_token key的dict) 初始化 SendSmsConfig
:param source:
:param api_server:
:param sys_name:
:param sys_token:
"""
if
source
is
None
:
source
=
dict
()
self
.
api_server
:
str
=
api_server
or
source
.
get
(
"api_server"
)
self
.
sys_name
:
str
=
sys_name
or
source
.
get
(
"sys_name"
)
self
.
sys_token
:
str
=
sys_token
or
source
.
get
(
"sys_token"
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
class
SendNoticeUtils
:
r
"""
发送通知 Utils 类,不依赖任何其他应用基础类
"""
send_email_config
:
SendEmailConfig
=
None
send_sms_config
:
SendSmsConfig
=
None
def
__init__
(
self
,
send_email_config
:
SendEmailConfig
=
None
,
send_sms_config
:
SendSmsConfig
=
None
):
r
"""
初始化
:param send_sms_config:
:param send_email_config:
"""
self
.
send_email_config
=
send_email_config
self
.
send_sms_config
=
send_sms_config
def
gen_to_receive_notice_user_list
(
self
,
user_list
:
list
)
->
list
:
r
"""
根据 source(包含 phone 和 email key的dict) 生成 ToReceiveNoticeUser 列表
:param user_list:
"""
if
not
user_list
:
return
list
()
result_list
=
list
()
for
source
in
user_list
:
if
isinstance
(
source
,
ToReceiveNoticeUser
):
tmp_source
=
source
else
:
tmp_source
=
ToReceiveNoticeUser
(
source
)
result_list
.
append
(
tmp_source
)
return
result_list
def
do_notice
(
self
,
to_receive_notice_user_list
:
list
,
subject
:
str
=
None
,
notice_content
:
str
=
None
,
add_subject_to_notice_content
:
bool
=
False
)
->
NoticeSendResult
:
r
"""
通知(短信通知,邮件通知等)
:param to_receive_notice_user_list:
:param subject:
:param notice_content:
:param add_subject_to_notice_content:
"""
to_receive_notice_user_list
=
self
.
gen_to_receive_notice_user_list
(
to_receive_notice_user_list
)
email_list
=
[
tmp_to_receive_notice_user
.
email
for
tmp_to_receive_notice_user
in
to_receive_notice_user_list
]
phone_list
=
[
tmp_to_receive_notice_user
.
phone
for
tmp_to_receive_notice_user
in
to_receive_notice_user_list
]
real_notice_content
=
notice_content
if
add_subject_to_notice_content
:
real_notice_content
=
Utility
.
join_str
(
subject
,
'
\n
'
,
notice_content
)
send_success_for_send_email
:
bool
=
self
.
do_send_email
(
email_list
,
subject
,
real_notice_content
)
send_success_for_send_sms
:
bool
=
self
.
do_send_sms
(
phone_list
,
real_notice_content
)
result
=
NoticeSendResult
()
result
.
sms_send_result
=
send_success_for_send_sms
result
.
email_send_result
=
send_success_for_send_email
return
result
def
do_notice_with_any_success
(
self
,
to_receive_notice_user_list
:
list
,
subject
:
str
=
None
,
notice_content
:
str
=
None
,
add_subject_to_notice_content
:
bool
=
False
)
->
bool
:
r
"""
通知(短信通知,邮件通知等)
:param to_receive_notice_user_list:
:param subject:
:param notice_content:
:param add_subject_to_notice_content:
"""
return
self
.
do_notice
(
to_receive_notice_user_list
,
subject
,
notice_content
,
add_subject_to_notice_content
).
is_any_success
()
def
do_send_email
(
self
,
to_mails
:
list
,
subject
:
str
=
None
,
notice_content
:
Union
[
str
,
Message
]
=
None
)
->
bool
:
r
"""
发送邮件
:param to_mails:
:param subject:
:param notice_content:
"""
smtp
=
None
try
:
if
self
.
send_email_config
is
None
:
raise
RuntimeError
(
'send_email_config为空'
)
mail_host
=
self
.
send_email_config
.
mail_host
mail_host_port
=
self
.
send_email_config
.
mail_host_port
mail_from_account
=
self
.
send_email_config
.
mail_from_account
mail_from_account_dwp
=
self
.
send_email_config
.
mail_from_account_dwp
if
not
mail_host
:
raise
RuntimeError
(
'send_email_config.mail_host 为空'
)
if
not
mail_host_port
:
raise
RuntimeError
(
'send_email_config.mail_host_port 为空'
)
if
not
mail_from_account
:
raise
RuntimeError
(
'send_email_config.mail_from_account 为空'
)
if
not
mail_from_account_dwp
:
raise
RuntimeError
(
'send_email_config.mail_from_account_dwp 为空'
)
smtp
=
smtplib
.
SMTP
(
host
=
mail_host
,
port
=
mail_host_port
)
smtp
.
login
(
mail_from_account
,
mail_from_account_dwp
)
real_notice_content
=
notice_content
if
not
isinstance
(
notice_content
,
Message
):
real_notice_content
=
MIMEText
(
MyStringUtils
.
to_str
(
notice_content
),
"plain"
,
"utf-8"
)
real_notice_content
[
'Subject'
]
=
Header
(
subject
,
'utf-8'
)
smtp
.
sendmail
(
mail_from_account
,
to_mails
,
real_notice_content
.
as_string
())
return
True
except
BaseException
as
e
:
logging
.
error
(
"发送邮件时发生异常"
)
logging
.
exception
(
e
,
exc_info
=
True
)
return
False
finally
:
if
smtp
is
not
None
:
smtp
.
quit
()
def
do_send_sms
(
self
,
phone_list
:
list
,
notice_content
:
str
=
None
)
->
bool
:
r
"""
发送邮件
:param phone_list:
:param notice_content:
"""
try
:
if
self
.
send_sms_config
is
None
:
raise
RuntimeError
(
'send_sms_config为空'
)
api_server
=
self
.
send_sms_config
.
api_server
sys_name
=
self
.
send_sms_config
.
sys_name
sys_token
=
self
.
send_sms_config
.
sys_token
if
not
api_server
:
raise
RuntimeError
(
'send_sms_config.api_server 为空'
)
if
not
sys_name
:
raise
RuntimeError
(
'send_sms_config.sys_name 为空'
)
if
not
sys_token
:
raise
RuntimeError
(
'send_sms_config.sys_token 为空'
)
schema_tuple
=
(
"http://"
,
"https://"
)
default_schema
=
'http://'
if
not
api_server
.
lower
().
startswith
(
schema_tuple
):
api_server
=
Utility
.
join_str
(
default_schema
,
api_server
)
map_header
=
dict
()
map_header
[
'Content-Type'
]
=
'application/json'
request_dict
=
dict
()
request_dict
[
'sysName'
]
=
sys_name
request_dict
[
'token'
]
=
sys_token
request_dict
[
'content'
]
=
notice_content
request_dict
[
'phoneNoList'
]
=
phone_list
resp
=
requests
.
post
(
api_server
,
data
=
Utility
.
dict2jsonstr
(
request_dict
),
headers
=
map_header
,
verify
=
False
)
# 判断状态码
if
resp
.
status_code
!=
200
:
raise
RuntimeError
(
Utility
.
join_str
(
f
"调用sms ApiServer接口返回状态码为
{
resp
.
status_code
}
"
))
# 判断RetCode
resp_dict
=
Utility
.
jsonstr2dict
(
resp
.
text
)
if
resp_dict
is
None
:
raise
RuntimeError
(
Utility
.
join_str
(
"调用sms ApiServer接口返回内容不是jsonObject,返回内容为: "
,
resp
.
text
))
ret_code
=
resp_dict
.
get
(
'RetCode'
)
if
str
(
ret_code
)
!=
'1'
:
raise
RuntimeError
(
Utility
.
join_str
(
"调用sms ApiServer接口返回RetCode不是1, 返回内容为: "
,
resp
.
text
))
return
True
except
BaseException
as
e
:
logging
.
error
(
"发送短信时发生异常"
)
logging
.
exception
(
e
,
exc_info
=
True
)
return
False
cucc_common_pkg/ludq_utils/util_methods.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
logging
from
typing
import
Optional
,
List
from
.app_exception
import
AppException
from
.base_const
import
ConstResponseCode
from
.common_app_config
import
CommonAppConfig
class
UtilMethods
:
@
classmethod
def
get_intvalue_from_dict_and_check_range
(
cls
,
source_dict
:
dict
,
key_name
:
str
,
min_value
:
Optional
[
int
]
=
None
,
max_value
:
Optional
[
int
]
=
None
,
other_allowed_values
:
List
[
int
]
=
None
,
action_desc
:
str
=
None
,
check_null
:
bool
=
True
,
default_value
:
int
=
0
,
)
->
int
:
r
"""
从字典中获取指定key的值,并检查其是否是int,并检查其值的范围
:param source_dict:
:param key_name:
:param min_value:可选值,提供则判断最小值,
:param max_value:可选值
:param other_allowed_values: 允许的值范围,例如 [-1], 在此范围内则不校验 最小/最大值
:param action_desc: 抛出异常时的描述
:param check_null: 默认为True,即值为None是抛出异常
:param default_value: 默认值, None表示0
"""
action_desc
=
action_desc
or
"从字典中获取字符串值"
if
not
key_name
:
result
=
source_dict
else
:
result
=
source_dict
.
get
(
key_name
)
if
result
is
None
:
if
check_null
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result
}
)为空"
)
else
:
result
=
default_value
if
default_value
is
not
None
else
0
if
not
isinstance
(
result
,
int
):
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_SYS_ERROR
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
,使用默认值
{
result
}
,但不是int类型"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result
}
)为空,使用默认值
{
result
}
,但不是int类型"
)
elif
not
isinstance
(
result
,
int
):
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
不是int类型"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result
}
)不是int类型"
)
if
other_allowed_values
and
result
in
other_allowed_values
:
return
result
if
min_value
is
not
None
and
result
<
min_value
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
的值小于
{
min_value
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result
}
)小于
{
min_value
}
"
)
if
max_value
is
not
None
and
result
>
max_value
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
的值大于
{
max_value
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result
}
)大于
{
max_value
}
"
)
return
result
@
classmethod
def
get_strvalue_from_dict
(
cls
,
source_dict
:
dict
,
key_name
:
str
,
check_null_or_empty
:
bool
=
True
,
default_value
:
str
=
None
,
action_desc
:
str
=
None
)
->
str
:
r
"""
从字典获取字符串,
:param source_dict:
:param key_name:
:param check_null_or_empty:是否检查值是否为空或null,默认为True
:param default_value: 默认值,默认为空字符串
:param action_desc: 抛出异常时的描述
"""
if
not
key_name
:
result_obj
=
source_dict
else
:
result_obj
=
source_dict
.
get
(
key_name
)
if
result_obj
is
not
None
:
result_obj
=
f
"
{
result_obj
}
"
if
check_null_or_empty
and
not
result_obj
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据为空"
)
return
result_obj
or
default_value
or
""
@
classmethod
def
get_boolvalue_from_dict
(
cls
,
source_dict
:
dict
,
key_name
:
str
,
check_null
:
bool
=
True
,
default_value
:
bool
=
False
,
action_desc
:
str
=
None
)
->
bool
:
r
"""
从字典获取bool值, 注意 default_value 是 None 时, 默认值为 None
:param source_dict:
:param key_name:
:param check_null:是否检查值是否为空或null,默认为True
:param default_value: 默认值,默认为 False
:param action_desc: 抛出异常时的描述
"""
if
not
key_name
:
result_obj
=
source_dict
else
:
result_obj
=
source_dict
.
get
(
key_name
)
if
result_obj
is
None
:
if
check_null
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result_obj
}
)为None"
)
else
:
result_obj
=
default_value
if
not
isinstance
(
result_obj
,
bool
):
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
不是bool类型"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result_obj
}
)不是bool类型"
)
return
result_obj
@
classmethod
def
get_objvalue_from_dict
(
cls
,
source_dict
:
dict
,
key_name
:
str
,
obj_class
,
action_desc
:
str
=
None
,
check_null
:
bool
=
True
,
**
kwargs
):
r
"""
从字典获取对象,
:param source_dict:
:param key_name:
:param obj_class:
:param action_desc:
:param check_null:
:param kwargs:
"""
if
not
key_name
:
result_obj
=
source_dict
else
:
result_obj
=
source_dict
.
get
(
key_name
)
if
isinstance
(
result_obj
,
obj_class
):
return
result_obj
elif
isinstance
(
result_obj
,
dict
):
kwargs
.
update
(
result_obj
)
return
obj_class
().
init_by_create_dict
(
**
kwargs
)
elif
result_obj
is
None
:
if
check_null
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result_obj
}
)为空"
)
else
:
return
None
else
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
为不支持的类型"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result_obj
}
)为不支持的类型"
)
@
classmethod
def
get_enumvalue_from_dict
(
cls
,
source_dict
:
dict
,
key_name
:
str
,
obj_class
,
action_desc
:
str
=
None
,
logger
:
logging
.
Logger
=
None
,
check_null
:
bool
=
True
,
default_value
=
None
,
):
r
"""
从字典获取枚举类型对象
:param source_dict:
:param key_name:
:param obj_class
:param action_desc:
:param logger:
:param check_null:
:param default_value:
"""
logger
=
logger
or
CommonAppConfig
().
common_logger
if
not
key_name
:
result_obj
=
source_dict
else
:
result_obj
=
source_dict
.
get
(
key_name
)
if
isinstance
(
result_obj
,
obj_class
):
return
result_obj
elif
result_obj
is
None
:
if
check_null
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result_obj
}
)为None"
)
else
:
return
default_value
else
:
try
:
return
obj_class
(
str
(
result_obj
))
except
BaseException
as
e
:
encountered_e
=
e
if
encountered_e
:
logger
.
error
(
encountered_e
)
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
不是有效的
{
obj_class
.
__name__
}
值类型"
).
with_traceback
(
encountered_e
.
__traceback__
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
result_obj
}
)不是有效的
{
obj_class
.
__name__
}
值类型"
).
with_traceback
(
encountered_e
.
__traceback__
)
@
classmethod
def
get_listvalue_from_dict
(
cls
,
source_dict
:
dict
,
key_name
:
str
,
obj_class
,
check_null
:
bool
=
False
,
action_desc
:
str
=
None
)
->
list
:
r
"""
从字典获取列表,
:param source_dict:
:param key_name:
:param obj_class
:param check_null: 默认为False
:param action_desc:
"""
if
not
key_name
:
tmp_key_value
=
source_dict
else
:
tmp_key_value
=
source_dict
.
get
(
key_name
)
if
not
tmp_key_value
:
if
check_null
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时未提供
{
key_name
}
"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据为空"
)
else
:
return
list
()
if
not
isinstance
(
tmp_key_value
,
list
):
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时提供
{
key_name
}
不是列表"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
tmp_key_value
}
)不是列表"
)
result_obj
=
list
()
for
tmp_obj
in
tmp_key_value
:
if
isinstance
(
tmp_obj
,
obj_class
):
result_obj
.
append
(
tmp_obj
)
elif
isinstance
(
tmp_obj
,
dict
):
result_obj
.
append
(
obj_class
().
init_by_create_dict
(
**
tmp_obj
))
else
:
if
key_name
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时
{
key_name
}
中的对象是不支持的类型"
)
else
:
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
f
"
{
action_desc
}
时源数据(
{
tmp_key_value
}
)中的对象是不支持的类型"
)
return
result_obj
cucc_common_pkg/ludq_utils/utils_base.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
import
base64
import
hashlib
import
json
import
logging
import
os
import
pickle
import
platform
import
random
import
re
import
string
import
subprocess
import
tempfile
import
time
import
traceback
import
urllib.parse
from
datetime
import
datetime
from
typing
import
Union
,
Optional
,
Dict
,
List
import
yaml
from
flask
import
Request
from
jinja2
import
Environment
,
FileSystemLoader
,
StrictUndefined
from
jinja2.defaults
import
VARIABLE_START_STRING
,
VARIABLE_END_STRING
from
.app_exception
import
AppRuntimeException
,
AppException
from
.app_response
import
AppResponse
from
.base_const
import
ConstResponseCode
from
.common_app_config
import
CommonAppConfig
from
.logined_user
import
LoginedUserInfo
from
.send_notices_utils
import
SendNoticeUtils
from
..globalutility
import
Utility
from
..my_stringutils
import
MyStringUtils
try
:
import
thread
except
ImportError
:
import
_thread
as
thread
class
CrnParseResult
:
product_code
:
Optional
[
str
]
=
None
region_code
:
Optional
[
str
]
=
None
acct_id
:
Optional
[
str
]
=
None
first_id_type
:
Optional
[
str
]
=
None
first_id
:
Optional
[
str
]
=
None
second_id_type
:
Optional
[
str
]
=
None
second_id
:
Optional
[
str
]
=
None
third_id_type
:
Optional
[
str
]
=
None
third_id
:
Optional
[
str
]
=
None
other_id_types
:
Optional
[
List
[
str
]]
=
None
other_ids
:
Optional
[
List
[
str
]]
=
None
first_crn
:
Optional
[
str
]
=
None
second_crn
:
Optional
[
str
]
=
None
third_crn
:
Optional
[
str
]
=
None
other_crns
:
Optional
[
List
[
str
]]
=
None
def
__init__
(
self
,
**
kwargs
):
self
.
product_code
=
kwargs
.
get
(
"product_code"
)
self
.
region_code
=
kwargs
.
get
(
"region_code"
)
self
.
acct_id
=
kwargs
.
get
(
"acct_id"
)
self
.
first_id_type
=
kwargs
.
get
(
"first_id_type"
)
self
.
first_id
=
kwargs
.
get
(
"first_id"
)
self
.
second_id_type
=
kwargs
.
get
(
"second_id_type"
)
self
.
second_id
=
kwargs
.
get
(
"second_id"
)
self
.
third_id_type
=
kwargs
.
get
(
"third_id_type"
)
self
.
third_id
=
kwargs
.
get
(
"third_id"
)
self
.
other_id_types
=
kwargs
.
get
(
"other_id_types"
)
self
.
other_ids
=
kwargs
.
get
(
"other_ids"
)
self
.
first_crn
=
kwargs
.
get
(
"first_crn"
)
self
.
second_crn
=
kwargs
.
get
(
"second_crn"
)
self
.
third_crn
=
kwargs
.
get
(
"third_crn"
)
self
.
other_crns
=
kwargs
.
get
(
"other_crns"
)
def
__str__
(
self
):
return
Utility
.
dict2jsonstr
(
self
.
__dict__
)
class
UtilityBaseV2
(
Utility
):
r
"""
便捷功能类,不依赖任何类的应用的便捷功能类
"""
def
__init__
(
self
,
logger
:
logging
.
Logger
=
None
):
self
.
logger
=
logger
or
CommonAppConfig
().
common_logger
@
classmethod
def
gen_logger
(
cls
,
logger_name
:
str
=
"not_named_logger"
,
log_level
:
int
=
None
,
):
r
"""
生成logger
:param logger_name: 通常可以用 self.__class__.__name__ 来提供
:param log_level: 默认为 logging.WARNING , 默认为 CommonAppConfig().log_level
"""
logger
=
logging
.
getLogger
(
logger_name
)
if
log_level
is
None
:
log_level
=
CommonAppConfig
().
log_level
logger
.
setLevel
(
log_level
)
if
not
logger
.
handlers
:
ch
=
logging
.
StreamHandler
()
ch
.
setLevel
(
logging
.
DEBUG
)
formatter
=
logging
.
Formatter
(
'%(asctime)s %(name)s - %(pathname)s - func:%(funcName)s - lineno:%(lineno)s - %(levelname)s - %(message)s'
,
datefmt
=
'%Y-%m-%d %H:%M:%S'
)
ch
.
setFormatter
(
formatter
)
logger
.
addHandler
(
ch
)
return
logger
@
classmethod
def
dict2jsonstr
(
cls
,
a_dict
:
dict
,
value_when_obj_is_none
:
str
=
'null'
,
sort_keys
:
bool
=
False
,
remove_key_of_none_value
:
bool
=
False
,
attr_name_when_value_is_obj
:
str
=
'__dict__'
)
->
str
:
r
"""
复用父类的 dict2jsonstr 的基础上,增加按排序key的方式生成json字符串,并且生成的字符串为最紧密的字符串
:param a_dict:
:param value_when_obj_is_none:
:param sort_keys
:param remove_key_of_none_value
:param attr_name_when_value_is_obj
:return:
"""
if
a_dict
is
None
:
return
value_when_obj_is_none
elif
not
isinstance
(
a_dict
,
dict
):
raise
ValueError
(
'参数类型不是字典'
)
if
remove_key_of_none_value
:
# 遍历dict的所有key,并递归删除value为None的key
a_dict
=
cls
.
dict_after_remove_key_of_none_value
(
a_dict
,
attr_name_when_value_is_obj
)
sort_keys
=
True
if
sort_keys
else
False
return
json
.
dumps
(
a_dict
,
ensure_ascii
=
False
,
default
=
lambda
o
:
cls
.
gen_dict_for_a_obj
(
o
,
attr_name_when_value_is_obj
),
sort_keys
=
sort_keys
,
separators
=
(
','
,
':'
))
@
classmethod
def
gen_dict_for_a_obj
(
cls
,
a_obj
,
attr_name
):
r
"""
主要用于json序列化时,value为对象时如何序列化,a_obj指定的属性或方法必须返回一个dict
:param a_obj:
:param attr_name:
:return:
"""
if
not
attr_name
:
return
getattr
(
a_obj
,
"__dict__"
,
None
)
if
isinstance
(
attr_name
,
str
):
if
hasattr
(
a_obj
,
attr_name
):
attr_obj
=
getattr
(
a_obj
,
attr_name
,
None
)
if
'__call__'
in
dir
(
attr_obj
):
return
attr_obj
()
else
:
return
attr_obj
else
:
return
getattr
(
a_obj
,
"__dict__"
,
None
)
else
:
if
'__call__'
in
dir
(
attr_name
):
return
attr_name
(
a_obj
)
else
:
return
getattr
(
a_obj
,
"__dict__"
,
None
)
@
classmethod
def
dict_after_remove_key_of_none_value
(
cls
,
a_dict
:
dict
,
attr_name_when_value_is_obj
:
str
=
'__dict__'
)
->
Optional
[
dict
]:
r
"""
遍历dict的所有key,并递归删除value为None的key
:param a_dict:
:param attr_name_when_value_is_obj
:return:
"""
if
a_dict
is
None
:
return
None
elif
not
isinstance
(
a_dict
,
dict
):
raise
ValueError
(
'参数类型不是字典'
)
result
=
dict
()
for
tmp_key
,
tmp_value
in
a_dict
.
items
():
if
tmp_value
is
None
:
continue
elif
isinstance
(
tmp_value
,
dict
):
result
[
tmp_key
]
=
cls
.
dict_after_remove_key_of_none_value
(
tmp_value
)
elif
isinstance
(
tmp_value
,
list
):
result
[
tmp_key
]
=
cls
.
list_after_remove_key_of_none_value
(
tmp_value
)
elif
isinstance
(
tmp_value
,
str
)
or
isinstance
(
tmp_value
,
int
)
or
isinstance
(
tmp_value
,
float
)
or
isinstance
(
tmp_value
,
bool
):
result
[
tmp_key
]
=
tmp_value
else
:
tmp_value
=
cls
.
dict_after_remove_key_of_none_value
(
cls
.
gen_dict_for_a_obj
(
tmp_value
,
attr_name_when_value_is_obj
))
if
tmp_value
is
not
None
:
result
[
tmp_key
]
=
tmp_value
return
result
@
classmethod
def
list_after_remove_key_of_none_value
(
cls
,
a_list
:
Union
[
list
,
tuple
],
attr_name_when_value_is_obj
:
str
=
'__dict__'
)
->
Optional
[
list
]:
r
"""
遍历list中每个dict的所有key,并递归删除value为None的key,list中的
:param a_list:
:param attr_name_when_value_is_obj
:return:
"""
if
a_list
is
None
:
return
None
elif
not
isinstance
(
a_list
,
(
list
,
tuple
)):
raise
ValueError
(
'参数类型不是list或元组'
)
result
=
list
()
for
tmp_value
in
a_list
:
if
tmp_value
is
None
\
or
isinstance
(
tmp_value
,
str
)
\
or
isinstance
(
tmp_value
,
int
)
\
or
isinstance
(
tmp_value
,
float
)
\
or
isinstance
(
tmp_value
,
bool
):
result
.
append
(
tmp_value
)
elif
isinstance
(
tmp_value
,
dict
):
result
.
append
(
cls
.
dict_after_remove_key_of_none_value
(
tmp_value
))
elif
isinstance
(
tmp_value
,
list
):
result
.
append
(
cls
.
list_after_remove_key_of_none_value
(
tmp_value
))
else
:
tmp_value
=
cls
.
dict_after_remove_key_of_none_value
(
cls
.
gen_dict_for_a_obj
(
tmp_value
,
attr_name_when_value_is_obj
))
result
.
append
(
tmp_value
)
return
result
@
classmethod
def
list2jsonstr
(
cls
,
a_list
:
Union
[
list
,
tuple
],
value_when_obj_is_none
:
str
=
'null'
,
sort_keys
:
bool
=
False
,
remove_key_of_none_value
:
bool
=
False
,
attr_name_when_value_is_obj
:
str
=
'__dict__'
)
->
str
:
r
"""
:param a_list:
:param value_when_obj_is_none:
:param sort_keys:
:param remove_key_of_none_value
:param attr_name_when_value_is_obj
:return:
"""
if
a_list
is
None
:
return
value_when_obj_is_none
elif
not
isinstance
(
a_list
,
(
list
,
tuple
)):
raise
ValueError
(
'参数类型不是list或元组'
)
if
remove_key_of_none_value
:
# 遍历dict的所有key,并递归删除value为None的key
a_list
=
cls
.
list_after_remove_key_of_none_value
(
a_list
,
attr_name_when_value_is_obj
)
sort_keys
=
True
if
sort_keys
else
False
return
json
.
dumps
(
a_list
,
ensure_ascii
=
False
,
default
=
lambda
o
:
cls
.
gen_dict_for_a_obj
(
o
,
attr_name_when_value_is_obj
),
sort_keys
=
sort_keys
,
separators
=
(
','
,
':'
))
def
generate_id_16
(
self
)
->
str
:
r
"""
生成一个 16 位的唯一数字编码
:return:
"""
# 产生10000 - 99999 的随机数
r1
=
10000
+
random
.
choice
(
range
(
90000
))
tmp_time
=
str
(
time
.
time
()).
replace
(
"."
,
""
)
length_of_tmp_time
=
len
(
tmp_time
)
while
length_of_tmp_time
<
13
:
tmp_time
=
str
(
time
.
time
()).
replace
(
"."
,
""
)
length_of_tmp_time
=
len
(
tmp_time
)
return
self
.
join_str
(
r1
,
tmp_time
[
length_of_tmp_time
-
13
:
length_of_tmp_time
-
2
])
def
generate_id_12
(
self
)
->
str
:
r
"""
生成一个 12 位的唯一数字编码
:return:
"""
# 产生10000 - 99999 的随机数
r1
=
10000
+
random
.
choice
(
range
(
90000
))
tmp_time
=
str
(
time
.
time
()).
replace
(
"."
,
""
)
length_of_tmp_time
=
len
(
tmp_time
)
while
length_of_tmp_time
<
9
:
tmp_time
=
str
(
time
.
time
()).
replace
(
"."
,
""
)
length_of_tmp_time
=
len
(
tmp_time
)
return
self
.
join_str
(
r1
,
tmp_time
[
length_of_tmp_time
-
9
:
length_of_tmp_time
-
2
])
def
get_cookies_dict_from_request
(
self
,
request
,
decoding
:
bool
=
True
)
->
dict
:
r
"""
使用比较原始的方法将request.headers.get('Cookie')中的内容放到一个dict[string,list[string]]结构中
:param request:
:param decoding:
:return:
"""
cookies_str
:
str
=
request
.
headers
.
get
(
'Cookie'
)
if
not
cookies_str
:
return
dict
()
all_cookies_list
:
list
=
cookies_str
.
split
(
';'
)
result_dict
=
dict
()
for
tmp_cookie_str
in
all_cookies_list
:
tmp_cookie_str_to_list
=
tmp_cookie_str
.
split
(
'='
)
tmp_cookie_key
=
tmp_cookie_str_to_list
[
0
].
lstrip
()
tmp_cookie_value
=
Utility
.
list_join_to_str
(
tmp_cookie_str_to_list
[
1
:],
separator_str
=
'='
)
if
decoding
:
tmp_cookie_key
=
urllib
.
parse
.
unquote
(
tmp_cookie_key
,
encoding
=
'utf-8'
)
tmp_cookie_value
=
urllib
.
parse
.
unquote
(
tmp_cookie_value
,
encoding
=
'utf-8'
)
tmp_original_value_list
=
result_dict
.
get
(
tmp_cookie_key
)
if
tmp_original_value_list
is
None
:
tmp_original_value_list
=
list
()
result_dict
[
tmp_cookie_key
]
=
tmp_original_value_list
tmp_original_value_list
.
append
(
tmp_cookie_value
)
return
result_dict
def
get_bearer_str_from_request
(
self
,
request
):
r
"""
从request的Header Bearer中获取bearer字符串
:param request:
:return:
"""
bearer_str
:
str
=
request
.
headers
.
get
(
'authorization'
)
if
not
bearer_str
:
return
None
else
:
start_str
=
'Bearer '
if
not
bearer_str
.
startswith
(
start_str
):
return
None
else
:
return
bearer_str
[
len
(
start_str
):]
def
is_mobile_number
(
self
,
mobile_str
:
str
)
->
bool
:
r
"""
检查字符串是否是手机格式
:param mobile_str
"""
return
re
.
match
(
'^1
\\
d{10}$'
,
MyStringUtils
.
to_str
(
mobile_str
))
is
not
None
def
is_email_address
(
self
,
email_str
:
str
)
->
bool
:
r
"""
检查字符串是否是邮箱格式
:param email_str
"""
return
re
.
match
(
'^([a-zA-Z0-9_
\\
-
\\
.]+)@((
\\
[[0-9]{1,3}
\\
.[0-9]{1,3}
\\
.[0-9]{1,3}
\\
.)|(([a-zA-Z0-9
\\
-]+
\\
.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(
\\
]?)$'
,
MyStringUtils
.
to_str
(
email_str
))
is
not
None
def
deep_dict_copy
(
self
,
a_dict
:
dict
)
->
dict
:
r
"""
:param a_dict:
:return:
"""
result
:
dict
=
a_dict
.
copy
()
for
tmp_key
,
tmp_value
in
result
.
items
():
if
isinstance
(
tmp_value
,
dict
):
result
[
tmp_key
]
=
self
.
deep_dict_copy
(
tmp_value
)
if
isinstance
(
tmp_value
,
list
):
result
[
tmp_key
]
=
self
.
deep_list_copy
(
tmp_value
)
return
result
def
deep_list_copy
(
self
,
a_list
:
list
)
->
list
:
r
"""
:param a_list:
:return:
"""
result
:
list
=
a_list
.
copy
()
for
tmp_index
,
tmp_value
in
enumerate
(
result
):
if
isinstance
(
tmp_value
,
dict
):
result
[
tmp_index
]
=
self
.
deep_dict_copy
(
tmp_value
)
if
isinstance
(
tmp_value
,
list
):
result
[
tmp_index
]
=
self
.
deep_list_copy
(
tmp_value
)
return
result
def
do_notice_admin
(
self
,
**
kwargs
)
->
bool
:
r
"""
通知管理员
异步的方式发送,总是返回true
:key subject 标题,默认为None
:key notice_content 内容,不能为空
:key admin_list 接收人,使用app_config.admin_list 提供
:key runtime_env 使用app_config.runtime_env 提供
:key send_email_config 使用app_config.send_email_config 提供
"""
admin_list
=
kwargs
.
get
(
"admin_list"
)
if
not
admin_list
:
return
True
runtime_env
=
kwargs
.
get
(
"runtime_env"
)
send_email_config
=
kwargs
.
get
(
"send_email_config"
)
subject
=
kwargs
.
get
(
"subject"
)
notice_content
=
kwargs
.
get
(
"notice_content"
)
def
run
():
real_subject
=
self
.
join_str
(
subject
,
'(运行时环境:'
,
runtime_env
,
')'
)
send_notice_utils
=
SendNoticeUtils
(
send_email_config
=
send_email_config
)
send_notice_utils
.
do_send_email
(
to_mails
=
admin_list
,
subject
=
real_subject
,
notice_content
=
notice_content
)
thread
.
start_new_thread
(
run
,
())
return
True
def
common_operation
(
self
,
retry_times_when_exception
,
operation_func
,
on_operation_failed
,
on_operation_completed
,
*
args
,
**
kwargs
):
r
"""
操作失败后重试的通用流程
:key retry_times_when_exception
:key operation_func
:key on_operation_completed
:key on_operation_failed
:return:
"""
has_attempt_times
=
0
encountered_exception
:
Optional
[
BaseException
]
=
None
result
:
Optional
[
bool
]
=
None
while
has_attempt_times
<=
retry_times_when_exception
:
# 按需重新初始化jedisPool
# 即上次循环中如果发生了异常,则在此次尝试执行操作前先重新初始化
if
encountered_exception
is
not
None
:
try
:
on_operation_failed
()
except
BaseException
as
e
:
# 如果重新初始化失败,直接结束并抛出异常
encountered_exception
=
e
break
# 尝试执行操作
try
:
result
=
operation_func
(
*
args
,
**
kwargs
)
except
AppRuntimeException
as
e
:
encountered_exception
=
e
break
except
BaseException
as
e
:
encountered_exception
=
e
# 如果没有遇到异常,直接结束循环
if
not
encountered_exception
:
break
# 尝试次数+1
has_attempt_times
+=
1
on_operation_completed
()
if
not
encountered_exception
:
return
result
else
:
raise
encountered_exception
def
get_container_name_from_container_image
(
self
,
container_image
:
str
):
r
"""
从形如 abc.com/istio/tgdevops/tgdevopspyservice:prod-1.0 的uri中提取 tgdevopspyservice
:param container_image:
:return:
"""
sections_of_image
=
container_image
.
split
(
":"
)
if
len
(
sections_of_image
)
==
1
:
str_contains_name
:
str
=
container_image
else
:
str_contains_name
:
str
=
self
.
list_join_to_str
(
sections_of_image
[
0
:
len
(
sections_of_image
)
-
1
],
separator_str
=
"/"
)
sections_of_image
=
str_contains_name
.
split
(
"/"
)
return
sections_of_image
[
len
(
sections_of_image
)
-
1
]
def
dict_update
(
self
,
to_update_dict
,
update_dict
):
r
"""
按深度更新字典,尽量保留原字典的内容
列表中不想更新的位置要跳过去需要提供占位符,使用字符串 __placeholder_for_update__ 表示
:param to_update_dict: 被更新的字典
:param update_dict: 提供更新内容的字典
:return:
"""
for
key
,
value
in
update_dict
.
items
():
original_value
=
to_update_dict
[
key
]
# 如果更新字典的值不是list,也不是字典,则直接设置更新字典中的内容
# 或者如果原字典中不包含这个key,则直接设置更新字典中的内容
# 或者如果原字典中包含这个key,但是值不是list,也不是字典,则直接设置更新字典中的内容
# 或者原字典中的值的类型和更新字典中的值的类型不一致,则直接设置更新字典中的内容
if
(
not
isinstance
(
value
,
dict
)
and
not
isinstance
(
value
,
list
))
\
or
key
not
in
to_update_dict
\
or
(
not
isinstance
(
original_value
,
dict
)
and
not
isinstance
(
original_value
,
list
))
\
or
type
(
original_value
)
!=
type
(
value
):
to_update_dict
[
key
]
=
value
elif
isinstance
(
value
,
dict
):
self
.
dict_update
(
original_value
,
value
)
else
:
self
.
list_update
(
original_value
,
value
)
def
list_update
(
self
,
to_update_list
,
update_list
):
r
"""
按深度更新字典,尽量保留原字典的内容,
列表更新和字典不一样,不想更新的位置要跳过去需要提供占位符,使用字符串 __placeholder_for_update__ 表示
:param to_update_list: 被更新的列表
:param update_list: 提供更新内容的列表,不想更新的index的值使用 __placeholder_for_update__ 表示
:return:
"""
len_of_to_update_list
=
len
(
to_update_list
)
for
index
,
value
in
enumerate
(
update_list
):
if
index
>=
len_of_to_update_list
:
return
if
value
==
"__placeholder_for_update__"
:
continue
original_value
=
to_update_list
[
index
]
# 如果更新列表的值不是list,也不是字典,则直接设置更新字典中的内容
# 或者如果原字典中不包含这个key,则直接设置更新字典中的内容
# 或者如果原字典中包含这个key,但是值不是list,也不是字典,则直接设置更新字典中的内容
# 或者原字典中的值的类型和更新字典中的值的类型不一致,则直接设置更新字典中的内容
if
(
not
isinstance
(
value
,
dict
)
and
not
isinstance
(
value
,
list
))
\
or
(
not
isinstance
(
original_value
,
dict
)
and
not
isinstance
(
original_value
,
list
))
\
or
type
(
original_value
)
!=
type
(
value
):
to_update_list
[
index
]
=
value
elif
isinstance
(
value
,
dict
):
self
.
dict_update
(
original_value
,
value
)
else
:
self
.
list_update
(
original_value
,
value
)
def
replace_placeholder_in_str
(
self
,
a_str
:
str
,
**
kwargs
):
r
"""
替换字符串中的形如正则表达式${abc}的值为kwargs中的值,如果kwargs中不包含abc,则不进行替换
:param a_str:
:return:
"""
def
_get_value_for_matched_key
(
matched
):
key_in_kwargs
=
matched
.
group
(
1
)
if
key_in_kwargs
in
kwargs
:
result2
=
kwargs
[
key_in_kwargs
]
if
kwargs
is
not
None
:
result2
=
str
(
result2
)
else
:
result2
=
matched
.
group
()
return
result2
result
=
re
.
sub
(
"
\"\\
${{int:(.+?)}}
\"
"
,
_get_value_for_matched_key
,
a_str
)
return
re
.
sub
(
"
\\
${{(.+?)}}"
,
_get_value_for_matched_key
,
result
)
def
generate_random_str
(
self
,
randomlength
:
int
=
16
)
->
str
:
"""
生成一个指定长度的随机字符串,其中
string.digits=0123456789
string.ascii_letters=abcdefghigklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ
"""
str_list
=
[
random
.
choice
(
string
.
digits
+
string
.
ascii_letters
+
r
"""!@#$%^&*()"""
)
for
_
in
range
(
randomlength
)]
random_str
=
''
.
join
(
str_list
)
return
random_str
def
check_resp_status_code_and_content
(
self
,
action
,
detail_info
,
resp
,
is_dict_for_resp_text
:
bool
=
True
,
expect_http_status_code
:
int
=
200
,
**
kwargs
)
->
Union
[
Dict
,
List
]:
r
"""
通用的检查http请求返回的resp的状态码是否是200,且返回内容是否是json格式,并最终返回resp.text表示的字典
:key action
:key detail_info
:key resp
:key is_dict_for_resp_text 默认为True
:key expect_http_status_code 默认为200
:key other_expect_http_status_codes 其他期望的status_code值
:key action_desc
:return:
"""
# 判断状态码
other_expect_http_status_codes
=
kwargs
.
get
(
"other_expect_http_status_codes"
)
if
other_expect_http_status_codes
is
None
:
other_expect_http_status_codes
=
list
()
other_expect_http_status_codes
.
append
(
expect_http_status_code
)
action_desc
=
kwargs
.
get
(
"action_desc"
)
if
not
action_desc
:
action_desc
=
action
if
resp
.
status_code
not
in
other_expect_http_status_codes
:
raise
AppRuntimeException
(
Utility
.
join_str
(
action_desc
,
f
"失败,返回的状态码不是
{
other_expect_http_status_codes
}
"
),
Utility
.
join_str
(
action
,
f
"返回的状态码不是
{
other_expect_http_status_codes
}
,statusCode="
,
resp
.
status_code
,
" "
,
detail_info
))
# 判断返回内容
result_str
=
resp
.
text
if
is_dict_for_resp_text
:
resp_obj
=
Utility
.
jsonstr2dict
(
result_str
)
else
:
resp_obj
=
Utility
.
jsonstr2list
(
result_str
)
if
resp_obj
is
None
:
raise
AppRuntimeException
(
Utility
.
join_str
(
action_desc
,
"失败,返回的内容不是json格式"
),
Utility
.
join_str
(
action
,
f
"返回的内容不是json格式,content=
{
result_str
}
"
,
" "
,
detail_info
)
)
return
resp_obj
def
check_resp_status_code_and_code
(
self
,
action
,
detail_info
,
resp
,
expect_code_value
:
str
=
ConstResponseCode
.
CODE_OK
,
other_expect_code_values
:
list
=
None
,
b_check_data_in_app_response
:
bool
=
True
,
**
kwargs
):
r
"""
通用的检查http请求返回的resp的状态码是否是200,且返回内容为iam标准response,且其中的code为OK,且其中的data字段不为空,
并最终返回data字段
:key action:
:key detail_info:
:key resp:
:key expect_code_value:
:key other_expect_code_values:
:key b_check_data_in_app_response:
:key key_for_data:
:key key_for_message:
:key key_for_code:
:key action_desc:
:return:
"""
return
self
.
check_resp_status_code_and_code_and_return_app_response
(
action
,
detail_info
,
resp
,
expect_code_value
=
expect_code_value
,
other_expect_code_values
=
other_expect_code_values
,
b_check_data_in_app_response
=
b_check_data_in_app_response
,
**
kwargs
).
data
def
check_resp_status_code_and_code_and_return_app_response
(
self
,
action
,
detail_info
,
resp
,
expect_code_value
:
str
=
ConstResponseCode
.
CODE_OK
,
other_expect_code_values
:
list
=
None
,
b_check_data_in_app_response
:
bool
=
True
,
**
kwargs
)
->
AppResponse
:
r
"""
通用的检查http请求返回的resp的状态码是否是200,且返回内容为iam标准response,且其中的code为OK,且其中的data字段不为空,
并最终返回data字段
:key action:
:key detail_info:
:key resp:
:key expect_code_value:
:key other_expect_code_values:
:key b_check_data_in_app_response:
:key key_for_data:
:key key_for_message:
:key key_for_code:
:key action_desc:
:return:
"""
action_desc
=
kwargs
.
get
(
"action_desc"
)
if
not
action_desc
:
action_desc
=
action
resp_dict
=
self
.
check_resp_status_code_and_content
(
action
,
detail_info
,
resp
,
action_desc
=
action_desc
)
# 转换成标准的 AppResponse
key_for_code
=
kwargs
.
get
(
"key_for_code"
)
key_for_data
=
kwargs
.
get
(
"key_for_data"
)
key_for_message
=
kwargs
.
get
(
"key_for_message"
)
app_response
=
AppResponse
.
from_dict
(
resp_dict
,
key_for_data
=
key_for_data
,
key_for_msg
=
key_for_message
,
key_for_code
=
key_for_code
)
if
other_expect_code_values
is
None
:
other_expect_code_values
=
list
()
other_expect_code_values
.
append
(
expect_code_value
)
if
app_response
.
code
not
in
other_expect_code_values
:
raise
AppRuntimeException
(
Utility
.
join_str
(
action_desc
,
"失败,返回的code="
,
app_response
.
code
,
" message="
,
app_response
.
message
),
Utility
.
join_str
(
action
,
"返回的code="
,
app_response
.
code
,
" message="
,
app_response
.
message
,
" "
,
detail_info
))
iam_response_data
=
app_response
.
data
# 如果未获取到权限验证结果信息,直接抛出异常
if
not
iam_response_data
and
b_check_data_in_app_response
:
raise
AppRuntimeException
(
Utility
.
join_str
(
action_desc
,
"失败,返回的内容不包含data数据"
),
Utility
.
join_str
(
action
,
"返回的内容不包含data数据,content="
,
Utility
.
dict2jsonstr
(
resp_dict
),
" "
,
detail_info
))
return
app_response
def
parse_crn
(
self
,
crn
:
str
,
*
crn_list
)
->
CrnParseResult
:
r
"""
解析格式为 crn:ucs::{product_code}:{region_code}:{account_id}:first/{first_id}/second/{second_id} 的crn表达式
从中解析出 product_code,region_code,acct_id,first_id,second_id,third_id
:param crn:
:param crn_list: 多个crn,解析第一个不为空的crn
"""
if
not
crn
:
for
tmp_crn
in
crn_list
:
if
tmp_crn
:
crn
=
tmp_crn
break
if
not
crn
:
raise
ValueError
(
f
"crn为空"
)
colon_parts
=
crn
.
split
(
":"
)
if
len
(
colon_parts
)
<
7
:
raise
ValueError
(
f
"
{
crn
}
不是有效的crn表达式"
)
product_code
=
colon_parts
[
3
]
region_code
=
colon_parts
[
4
]
acct_id
=
colon_parts
[
5
]
base_crn
=
":"
.
join
(
colon_parts
[
0
:
6
])
res_part
=
colon_parts
[
6
]
res_parts
=
res_part
.
split
(
"/"
)
len_parts
=
len
(
res_parts
)
first_id_type
=
None
first_id
=
None
second_id_type
=
None
second_id
=
None
third_id_type
=
None
third_id
=
None
other_id_types
=
list
()
other_ids
=
list
()
first_crn
=
None
second_crn
=
None
third_crn
=
None
other_crns
=
list
()
if
len_parts
>=
2
:
first_id_type
=
res_parts
[
0
]
first_id
=
res_parts
[
1
]
tmp_res_part
=
"/"
.
join
(
res_parts
[
0
:
2
])
first_crn
=
":"
.
join
([
base_crn
,
tmp_res_part
])
if
len_parts
>=
4
:
second_id_type
=
res_parts
[
2
]
second_id
=
res_parts
[
3
]
tmp_res_part
=
"/"
.
join
(
res_parts
[
0
:
4
])
second_crn
=
":"
.
join
([
base_crn
,
tmp_res_part
])
if
len_parts
>=
6
:
third_id_type
=
res_parts
[
4
]
third_id
=
res_parts
[
5
]
tmp_res_part
=
"/"
.
join
(
res_parts
[
0
:
6
])
third_crn
=
":"
.
join
([
base_crn
,
tmp_res_part
])
if
len_parts
>=
8
:
for
index
in
range
(
7
,
len_parts
):
if
index
%
2
!=
0
:
other_ids
.
append
(
res_parts
[
index
])
tmp_res_part
=
"/"
.
join
(
res_parts
[
0
:
index
+
1
])
other_crns
.
append
(
":"
.
join
([
base_crn
,
tmp_res_part
]))
else
:
other_id_types
.
append
(
res_parts
[
index
])
return
CrnParseResult
(
product_code
=
product_code
,
region_code
=
region_code
,
acct_id
=
acct_id
,
first_id_type
=
first_id_type
,
first_id
=
first_id
,
second_id_type
=
second_id_type
,
second_id
=
second_id
,
third_id_type
=
third_id_type
,
third_id
=
third_id
,
other_id_types
=
other_id_types
,
other_ids
=
other_ids
,
first_crn
=
first_crn
,
second_crn
=
second_crn
,
third_crn
=
third_crn
,
other_crns
=
other_crns
,
)
def
gen_crn_expression
(
self
,
product_code
:
str
,
region_code
:
str
,
account_id
:
str
,
res_part
:
str
)
->
str
:
r
"""
生成crn表达式, 示例 crn:ucs::cfc:gz-tst:12345:func/helloworld
:param product_code: 例如 cfc
:param region_code: 例如 gz-tst
:param account_id:
:param res_part:资源部分的描述字符串,例如 func/helloworld
"""
region_code
=
region_code
or
""
return
f
"crn:ucs::
{
product_code
}
:
{
region_code
}
:
{
account_id
}
:
{
res_part
}
"
def
rmdir_or_file
(
self
,
file_path
:
str
):
r
"""
根据操作系统执行删除文件夹/文件
:param file_path:
"""
if
not
os
.
path
.
exists
(
file_path
):
return
if
os
.
path
.
isdir
(
file_path
):
# 执行删除文件夹操作
if
platform
.
system
()
==
'Windows'
:
subprocess
.
call
(
f
"rmdir /S /Q
{
file_path
}
"
,
shell
=
True
)
else
:
subprocess
.
call
(
f
"rm -rf
{
file_path
}
"
,
shell
=
True
)
elif
os
.
path
.
isfile
(
file_path
):
# 执行删除文件操作
os
.
remove
(
file_path
)
else
:
# 特殊文件,不执行操作
self
.
logger
.
warning
(
f
"执行删除操作时发现
{
file_path
}
is a special file(socket,FIFO,device file),不执行删除操作"
)
def
read_file_content
(
self
,
file_path
)
->
str
:
"""
读取文件内容
:param file_path: 文件路径
:return: str -> file-content
"""
with
open
(
file_path
,
'r'
)
as
f
:
# 打开一个文件,必须是'rb'模式打开
return
f
.
read
()
def
yaml_load
(
self
,
yaml_str
:
str
)
->
Union
[
dict
,
str
]:
r
"""
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
"""
dict_list
=
self
.
yaml_load_list
(
cfg_content
=
yaml_str
)
if
dict_list
:
return
dict_list
[
0
]
else
:
return
dict
()
def
yaml_load_list
(
self
,
path
:
str
=
None
,
cfg_content
:
str
=
None
)
->
List
[
Union
[
Dict
,
str
]]:
"""
将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据,每个dict放到list中返回
无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
:param path: 文件路径
:param cfg_content:
:return: list -> []
"""
if
path
:
cfg
=
self
.
read_file_content
(
file_path
=
path
)
else
:
cfg
=
cfg_content
or
""
yaml_generator
=
yaml
.
safe_load_all
(
cfg
)
# 将yaml格式文件转换为dict值,该yaml文件可以包含多块yaml数据
if
yaml_generator
is
None
:
return
list
()
# 转成dict并保存到list中
yaml_list
=
list
()
for
one_yaml_generator
in
yaml_generator
:
if
one_yaml_generator
is
None
:
continue
yaml_list
.
append
(
json
.
loads
(
json
.
dumps
(
one_yaml_generator
)))
return
yaml_list
def
yaml_dump_list_to_str
(
self
,
source_list
:
List
[
Dict
])
->
str
:
r
"""
将字典列表序列化成为yaml字符串
"""
return
yaml
.
safe_dump_all
(
documents
=
source_list
,
encoding
=
'utf-8'
,
allow_unicode
=
True
).
decode
(
'utf-8'
)
def
yaml_dump_list_to_file
(
self
,
source_list
:
List
[
Dict
],
file_path
:
str
):
r
"""
将字典列表序列化成为yaml字符串
"""
with
open
(
file_path
,
'w'
,
encoding
=
'utf-8'
)
as
f
:
# 打开一个文件,必须是'rb'模式打开
yaml
.
safe_dump_all
(
documents
=
source_list
,
stream
=
f
,
encoding
=
'utf-8'
,
allow_unicode
=
True
)
def
dict_pop_key
(
self
,
source_dict
:
dict
,
key_name
:
str
):
r
"""
改进的移除dict的key,如果key不存在,则返回None,否则返回原始的 dict.pop(key)
:param source_dict:
:param key_name:
"""
return
source_dict
.
pop
(
key_name
)
if
key_name
in
source_dict
else
None
def
gen_user_info
(
self
,
login_user
:
LoginedUserInfo
,
output_type
:
str
=
"str"
)
->
Union
[
str
,
dict
]:
r
"""
生成 {"user_id":"","user_name":""}格式的字典或字符串
:param login_user:
:param output_type: str or dict, 输出格式,默认为 str
"""
output_type
=
output_type
or
"str"
result_dict
=
{
"user_id"
:
str
(
login_user
.
user_id
),
"user_name"
:
login_user
.
user_name
}
if
output_type
==
"str"
:
return
self
.
dict2jsonstr
(
result_dict
)
else
:
return
result_dict
def
get_content_after_render
(
self
,
dir_path
:
str
=
"./resource/"
,
encoding
:
str
=
"utf-8"
,
template_name
:
str
=
None
,
render_dict
:
dict
=
None
,
**
kwargs
)
->
str
:
r
"""
获取jinja2 template渲染后的文件内容
:param dir_path: 被渲染文件所在目录,默认为 ./resource/
:param encoding: 获取文件内容编码,默认为 utf-8
:param template_name: 模板文件名称
:param render_dict: 渲染模板用的字典
:key variable_start_string:
:key variable_end_string:
"""
dir_path
=
dir_path
or
"./resource/"
encoding
=
encoding
or
"utf-8"
render_dict
=
render_dict
or
dict
()
variable_start_string
=
kwargs
.
get
(
"variable_start_string"
)
or
VARIABLE_START_STRING
variable_end_string
=
kwargs
.
get
(
"variable_end_string"
)
or
VARIABLE_END_STRING
env
=
Environment
(
loader
=
FileSystemLoader
(
dir_path
,
encoding
=
encoding
),
variable_start_string
=
variable_start_string
,
variable_end_string
=
variable_end_string
,
undefined
=
StrictUndefined
,
)
# 创建文件加载器对象
template
=
env
.
get_template
(
template_name
)
# 获取一个模板文件
return
template
.
render
(
render_dict
)
# 渲染
def
gen_ip_port_str
(
self
,
schema
:
Optional
[
str
]
=
"http"
,
num_list
:
list
=
None
,
port
:
Optional
[
int
]
=
None
):
r
"""
生成url格式字符串,防止sonar检查
:param schema: 默认为http,None时不计入字符串内容
:param num_list: IP地址数字
:param port: 端口数字
"""
result_str
=
f
"
{
schema
}
://"
if
schema
else
""
str_list
=
[
str
(
num
)
for
num
in
num_list
]
result_str
+=
"."
.
join
(
str_list
)
if
port
is
not
None
:
result_str
+=
f
":
{
port
}
"
return
result_str
def
get_md5_of_string
(
self
,
src
:
str
)
->
str
:
"""
获取字符串的md5值
:param src:
:return:
"""
md1
=
hashlib
.
md5
()
# 创建一个md5算法对象
md1
.
update
(
src
.
encode
(
'UTF-8'
))
return
md1
.
hexdigest
()
def
get_md5_of_file
(
self
,
filepath
:
str
,
read_byte_once
:
int
=
8096
)
->
str
:
"""
获取文件的md5值
:param filepath:
:param read_byte_once:
:return:
"""
if
not
os
.
path
.
isfile
(
filepath
):
return
""
myhash
=
hashlib
.
md5
()
# 创建一个md5算法对象
with
open
(
filepath
,
'rb'
)
as
f
:
# 打开一个文件,必须是'rb'模式打开
while
True
:
b
=
f
.
read
(
read_byte_once
)
# 由于是一个文件,每次只读取固定字节
if
not
b
:
break
# 当读取内容不为空时对读取内容进行update
myhash
.
update
(
b
)
return
myhash
.
hexdigest
()
def
get_md5_of_dir
(
self
,
dirpath
:
str
)
->
str
:
"""
获取文件夹的md5值
:param dirpath:
:return:
"""
if
not
os
.
path
.
isdir
(
dirpath
):
return
""
dir_path_for_md5_file
=
tempfile
.
mkdtemp
()
md5_file
=
os
.
path
.
join
(
dir_path_for_md5_file
,
"tmp.md5"
)
with
open
(
md5_file
,
'w'
)
as
outfile
:
for
root
,
subdirs
,
files
in
os
.
walk
(
dir_path_for_md5_file
):
for
file
in
files
:
filefullpath
=
os
.
path
.
join
(
root
,
file
)
md5
=
self
.
get_md5_of_file
(
filefullpath
)
outfile
.
write
(
md5
)
val
=
self
.
get_md5_of_file
(
md5_file
)
self
.
rmdir_or_file
(
dir_path_for_md5_file
)
return
val
def
get_md5_of_file_or_dir
(
self
,
filepath
:
str
)
->
str
:
r
"""
获取文件/文件夹的md5值
:param filepath:文件/文件夹的路径
"""
if
os
.
path
.
isfile
(
filepath
):
return
self
.
get_md5_of_file
(
filepath
)
elif
os
.
path
.
isdir
(
filepath
):
return
self
.
get_md5_of_dir
(
filepath
)
else
:
return
""
def
base64_decode_with_padding
(
self
,
payload
:
Union
[
str
,
bytes
])
->
str
:
r
"""
base64解码,如果payload字节数不是4的倍数,会使用=补足
:param payload: 字符串或bytes
"""
if
isinstance
(
payload
,
str
):
byte_payload
=
payload
.
encode
(
"utf-8"
)
else
:
byte_payload
=
payload
missing_padding
=
4
-
len
(
byte_payload
)
%
4
if
missing_padding
:
byte_payload
+=
b
'='
*
missing_padding
b_result
=
base64
.
b64decode
(
byte_payload
)
return
str
(
b_result
,
encoding
=
"utf-8"
)
def
parse_jwt_str
(
self
,
jwt_str
:
str
)
->
str
:
r
"""
:param jwt_str: jwt字符串
"""
payload
=
jwt_str
.
split
(
"."
)[
1
]
return
self
.
base64_decode_with_padding
(
payload
)
def
attemp_parse_login_user_info
(
self
,
jwt_str
:
str
=
None
,
request
:
Request
=
None
)
->
Optional
[
LoginedUserInfo
]:
r
"""
尝试从jwt字符串中解析出登陆用户信息,解析失败返回None
:param jwt_str: jwt字符串
:param request: flask.Request对象
"""
if
not
jwt_str
and
request
:
jwt_str
=
request
.
cookies
.
get
(
"accessToken"
)
or
""
try
:
payload
=
self
.
parse_jwt_str
(
jwt_str
)
if
jwt_str
else
None
if
not
payload
:
raise
AppRuntimeException
(
message
=
"解析jwt字符串后获取的payload内容为空"
,
detail
=
f
"jwt_str=
{
jwt_str
}
"
,
)
payload_dict
=
self
.
jsonstr2dict
(
payload
)
if
not
payload_dict
:
raise
AppRuntimeException
(
message
=
"jwt的payload为空或者不是有效的json字符串"
,
detail
=
f
"payload=
{
payload
}
"
,
)
return
LoginedUserInfo
.
from_dict
(
payload_dict
,
jwt_str
)
except
BaseException
as
e
:
self
.
logger
.
error
(
e
)
exception_tracback
=
traceback
.
format_exc
()
self
.
logger
.
error
(
exception_tracback
)
return
None
def
retry_operation
(
self
,
operation_func
,
is_ok_function
,
is_ok_function_for_exception
=
None
,
operation_func_desc
:
str
=
None
,
max_execute_times
:
int
=
240
,
wait_seconds_for_retry
:
float
=
1
):
r
"""
通用的重试动作
:param operation_func: 操作函数,请使用 functools.partial 保证传入的函数在执行时不用再传入任何参数
:param is_ok_function: 对函数执行结果的判断函数,
该函数应返回一个Tuple,第一个元素为 bool 类型,表示对操作结果是否认为成功,第二个参数为真正的函数返回结果
:param is_ok_function_for_exception: 如果提供了该函数,则函数执行期间发生异常时使用该函数对异常进行判断,
该函数应返回一个Tuple,第一个元素为 bool 类型,表示发生异常时是否认为操作成功,第二个参数为认为成功时的函数返回结果
:param operation_func_desc: 操作函数描述,默认为 字面值"operation_func"
:param max_execute_times: 默认 240次
:param wait_seconds_for_retry: 默认 1 秒
"""
operation_func_desc
=
operation_func_desc
or
"operation_func"
has_executed_times
=
0
while
True
:
# 尝试执行,并获得结果或捕获异常
tmp_result
=
None
encountered_exception
=
None
try
:
tmp_result
=
operation_func
()
is_ok
,
tmp_result
=
is_ok_function
(
tmp_result
)
if
is_ok
:
return
tmp_result
except
BaseException
as
e
:
if
is_ok_function_for_exception
:
is_ok
,
tmp_result2
=
is_ok_function_for_exception
(
e
)
if
is_ok
:
return
tmp_result2
else
:
self
.
logger
.
error
(
f
"执行
{
operation_func_desc
}
时发生异常:
{
e
}
"
)
self
.
logger
.
error
(
traceback
.
format_exc
())
encountered_exception
=
e
else
:
self
.
logger
.
error
(
f
"执行
{
operation_func_desc
}
时发生异常:
{
e
}
"
)
self
.
logger
.
error
(
traceback
.
format_exc
())
encountered_exception
=
e
# 记录执行次数+1
has_executed_times
+=
1
# 判断是否已经达到最大执行次数
if
has_executed_times
>=
max_execute_times
>
0
:
# 返回最后一次执行的结果或抛出异常
if
encountered_exception
:
raise
encountered_exception
else
:
return
tmp_result
# 睡眠间隔
time
.
sleep
(
wait_seconds_for_retry
)
def
get_dict_from_dict_and_set_when_none
(
self
,
source_dict
:
dict
,
key_name
:
str
)
->
dict
:
r
"""
从字典中获取字典,如果指定的key不存在,则设置key对应的对象为一个空的字典
:param source_dict:
:param key_name:
"""
result_dict
=
source_dict
.
get
(
key_name
)
if
result_dict
is
None
:
result_dict
=
dict
()
source_dict
[
key_name
]
=
result_dict
return
result_dict
def
get_list_from_dict_and_set_when_none
(
self
,
source_dict
:
dict
,
key_name
:
str
)
->
list
:
r
"""
从字典中获取字典,如果指定的key不存在,则设置key对应的对象为一个空的字典
:param source_dict:
:param key_name:
"""
result_list
=
source_dict
.
get
(
key_name
)
if
result_list
is
None
:
result_list
=
list
()
source_dict
[
key_name
]
=
result_list
return
result_list
def
serialized
(
self
,
source_obj
)
->
str
:
r
"""
序列化为字符串,使用 pickle.dump 然后 base64编码, 然后以 utf-8 解码为字符串
"""
obj_src_bytes
=
pickle
.
dumps
(
source_obj
)
return
base64
.
b64encode
(
obj_src_bytes
).
decode
(
"utf-8"
)
def
reverse_serialized
(
self
,
source_str
:
str
):
r
"""
反序列化,将字符串以utf-8编码为bytes,然后base64解码,然后 pickle.loads为对象
"""
obj_src_bytes
=
base64
.
b64decode
(
source_str
.
encode
(
"utf-8"
))
return
pickle
.
loads
(
obj_src_bytes
)
def
parse_cpu_value_to_m
(
self
,
source_str
:
str
,
parse_empty_to_0
:
bool
=
False
,
)
->
int
:
"""
根据 10m 或 10 这样的字符串计算 cpu 有多少 m
"""
source_str_desc
=
source_str
or
"空字符串"
if
not
source_str
and
parse_empty_to_0
:
return
0
elif
source_str
.
endswith
(
"m"
):
return
int
(
source_str
[
0
:
-
1
])
elif
source_str
.
isdigit
():
# 全部是数字,则转换为m需要乘1000
return
int
(
float
(
source_str
)
*
1000
)
else
:
failed_reason
=
f
"无法将
{
source_str_desc
}
解析为单位为m的CPU核数"
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
failed_reason
,
)
def
parse_mem_value_to_b
(
self
,
source_str
:
str
,
parse_empty_to_0
:
bool
=
False
,
)
->
int
:
"""
根据 100Mi 或 10Gi 这样的字符串计算 mem 有多少字节
"""
source_str_desc
=
source_str
or
"空字符串"
if
not
source_str
and
parse_empty_to_0
:
return
0
elif
source_str
.
endswith
(
"Ki"
):
return
int
(
float
(
source_str
[
0
:
-
2
])
*
1024
**
1
)
elif
source_str
.
endswith
(
"Mi"
):
return
int
(
float
(
source_str
[
0
:
-
2
])
*
1024
**
2
)
elif
source_str
.
endswith
(
"Gi"
):
return
int
(
float
(
source_str
[
0
:
-
2
])
*
1024
**
3
)
elif
source_str
.
endswith
(
"Ti"
):
return
int
(
float
(
source_str
[
0
:
-
2
])
*
1024
**
4
)
elif
source_str
.
endswith
(
"Pi"
):
return
int
(
float
(
source_str
[
0
:
-
2
])
*
1024
**
5
)
elif
source_str
.
endswith
(
"Ei"
):
return
int
(
float
(
source_str
[
0
:
-
2
])
*
1024
**
6
)
elif
source_str
.
endswith
(
"k"
):
return
int
(
float
(
source_str
[
0
:
-
1
])
*
1000
**
1
)
elif
source_str
.
endswith
(
"M"
):
return
int
(
float
(
source_str
[
0
:
-
1
])
*
1000
**
2
)
elif
source_str
.
endswith
(
"G"
):
return
int
(
float
(
source_str
[
0
:
-
1
])
*
1000
**
3
)
elif
source_str
.
endswith
(
"T"
):
return
int
(
float
(
source_str
[
0
:
-
1
])
*
1000
**
4
)
elif
source_str
.
endswith
(
"P"
):
return
int
(
float
(
source_str
[
0
:
-
1
])
*
1000
**
5
)
elif
source_str
.
endswith
(
"E"
):
return
int
(
float
(
source_str
[
0
:
-
1
])
*
1000
**
6
)
elif
source_str
.
isdigit
():
# 全部是数字则单位为B
return
int
(
source_str
)
else
:
failed_reason
=
f
"无法将
{
source_str_desc
}
解析为内存字节数量"
raise
AppException
(
code
=
ConstResponseCode
.
CODE_MISSING_PARAMETER
,
message
=
failed_reason
,
)
def
format_time_to_utc_str
(
self
,
param_time
:
Union
[
int
,
float
,
str
,
datetime
])
->
Optional
[
str
]:
r
"""
将时间格式化成UTC字符串,格式为标准时区的 %Y-%m-%dT%H:%M:%SZ
:param param_time: 可以是unix时间戳(秒数),也可以是本地日期的字符串,或者是datetime类型的日期
"""
if
param_time
is
None
:
return
None
if
isinstance
(
param_time
,
(
int
,
float
)):
parsed_utctime
=
datetime
.
utcfromtimestamp
(
param_time
)
return
parsed_utctime
.
strftime
(
'%Y-%m-%dT%H:%M:%SZ'
)
elif
isinstance
(
param_time
,
datetime
):
parsed_utctime
=
datetime
.
utcfromtimestamp
(
param_time
.
timestamp
())
return
parsed_utctime
.
strftime
(
'%Y-%m-%dT%H:%M:%SZ'
)
elif
isinstance
(
param_time
,
str
):
if
not
param_time
:
return
None
parsed_time
=
self
.
parse_datestr
(
param_time
)
if
parsed_time
is
None
:
raise
ValueError
(
f
"param_time不是有效的时间字符串:
{
param_time
}
"
)
parsed_utctime
=
datetime
.
utcfromtimestamp
(
parsed_time
.
timestamp
())
return
parsed_utctime
.
strftime
(
'%Y-%m-%dT%H:%M:%SZ'
)
else
:
raise
ValueError
(
f
"param_time不能解析为时间:不支持的类型:
{
param_time
.
__class__
}
"
)
def
format_time_to_unix_timestamp_seconds
(
self
,
param_time
:
Union
[
int
,
float
,
str
,
datetime
])
->
Optional
[
float
]:
r
"""
将时间格式化成unix时间戳(秒数)
:param param_time: 可以是unix时间戳(秒数),也可以是本地日期的字符串,或者是datetime类型的日期
"""
if
param_time
is
None
:
return
None
if
isinstance
(
param_time
,
(
int
,
float
)):
return
param_time
elif
isinstance
(
param_time
,
datetime
):
return
param_time
.
timestamp
()
elif
isinstance
(
param_time
,
str
):
if
not
param_time
:
return
None
parsed_time
=
self
.
parse_datestr
(
param_time
)
if
parsed_time
is
None
:
raise
ValueError
(
f
"param_time不是有效的时间字符串:
{
param_time
}
"
)
return
parsed_time
.
timestamp
()
else
:
raise
ValueError
(
f
"param_time不能解析为unix时间戳:不支持的类型:
{
param_time
.
__class__
}
"
)
def
get_object_from_obj_by_paths
(
self
,
source_obj
,
key_paths
:
List
[
str
]
=
None
,
key_path_str
:
str
=
None
):
r
"""
按照key的路径依次获取指定路径序列的key对应的字典或列表,如果中途key miss,或index超出范围,则返回None,
:param source_obj: 列表或字典
:param key_paths: 优先级比 key_path_str 高
:param key_path_str: 格式为 components.ingressGateways.0.k8s.resources
"""
if
not
key_paths
:
key_path_str
=
key_path_str
or
""
key_paths
=
[
int
(
x
)
if
x
.
isdigit
()
else
x
for
x
in
key_path_str
.
strip
(
"."
).
split
(
"."
)]
result_obj
=
source_obj
for
tmp_key
in
key_paths
:
if
isinstance
(
tmp_key
,
int
):
if
len
(
result_obj
)
>
tmp_key
:
result_obj
=
result_obj
[
tmp_key
]
else
:
result_obj
=
None
else
:
result_obj
=
result_obj
.
get
(
tmp_key
)
if
result_obj
is
None
:
return
result_obj
return
result_obj
def
load_file_as_dict
(
self
,
filepath
:
str
)
->
dict
:
r
"""
判断文件后缀名, 如果是 .yaml 或 .yml 则按 yaml 方式读取,
文件内容不合规,则抛出异常
"""
with
open
(
filepath
,
'r'
,
encoding
=
"utf-8"
)
as
dict_file
:
conf_content
=
dict_file
.
read
()
# 无论是 yaml 格式 还是 json 格式,都可以使用 yaml_load 读取,但读取的结果可能是 字典或字符串
dict_content
=
self
.
yaml_load
(
conf_content
)
if
isinstance
(
dict_content
,
str
):
raise
ValueError
(
f
"
{
filepath
}
的内容不是有效的json或yaml字符串"
)
return
dict_content
cucc_common_pkg/ludq_utils/utils_get_cluster_info/__init__.py
0 → 100644
View file @
11b2601e
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Version: 1.0
@Python Version:3.6.6
@Author: ludq1
@Email: ludq1@chinaunicom.cn
@date: 2023/04/07 11:40:00
@Description:
"""
Prev
1
2
3
4
5
6
…
8
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment