diff --git a/release-assistant/javcra/api/obscloud.py b/release-assistant/javcra/api/obscloud.py index 4b9ee6a15d568fd04312160ed9c3e2335c2c3982..b99122e8eda57865cc5584358c9f01dc928da2ed 100644 --- a/release-assistant/javcra/api/obscloud.py +++ b/release-assistant/javcra/api/obscloud.py @@ -189,13 +189,13 @@ class ObsCloud: install_list = self.bucket_list("{}/{}/check_result/failed_log".format(prefix_name, branch)) for build_con in build_list: - if build_con.split("/")[3]: - build_res.add(build_con.split("/")[3]) + if build_con.split("/")[4]: + build_res.add(build_con.split("/")[4]) content_list["build_list"] = build_res for install_con in install_list: - if install_con.split("/")[3]: - install_res.add(install_con.split("/")[3]) + if install_con.split("/")[4]: + install_res.add(install_con.split("/")[4]) content_list["install_list"] = install_res return content_list @@ -227,8 +227,10 @@ class ObsCloud: failed_upload_file = [] for path in paths: path_name = path.replace(str(local_path), "") - res = self.upload_dir("{}/{}{}".format(prefix_name, branch, path_name), path) - if not res: + if not path_name.startswith("/"): + path_name = "/" + path_name + upload_res = self.upload_dir("{}/{}{}".format(prefix_name, branch, path_name), path) + if not upload_res: logging.error("%s File upload failed" % path_name) failed_upload_file.append(path) if failed_upload_file: @@ -249,7 +251,8 @@ if __name__ == '__main__': parser.add_argument("--bucketname", required=False, type=str, default="release-tools", help="Name of the branch") args = parser.parse_args() client = ObsCloud(args.ak, args.sk, args.server, args.bucketname) - res = client.run(args.branch, args.choice, args.path) - if not res: + RESP = client.run(args.branch, args.choice, args.path) + if not RESP: logging.error("File archiving failure") - logging.info("File archiving succeeded") + else: + logging.info("File archiving succeeded") diff --git a/release-assistant/javcra/application/checkpart/checktest.py b/release-assistant/javcra/application/checkpart/checktest.py index 8ed5e68fa7a56b429e57cf288b3ae182b550e432..82a68e01709be4909f4c0694b60aa4bfc90e126e 100644 --- a/release-assistant/javcra/application/checkpart/checktest.py +++ b/release-assistant/javcra/application/checkpart/checktest.py @@ -49,9 +49,9 @@ class CheckTest(Issue): for con in body.split("\n"): colon = ":" if ":" in con else ":" for role, people in ROLE_DICT.items(): - if people not in con: + if people + colon not in con: continue - personnel_access[role] = con.split(colon)[1] + personnel_access[role] = con.split(people + colon)[1] return personnel_access except IndexError as error: logger.error("Error parsing issue description information %s" % error) diff --git a/release-assistant/javcra/application/modifypart/modifyentrance.py b/release-assistant/javcra/application/modifypart/modifyentrance.py index 7aa111d19eb57997e352d491932214644190981c..85a400221b7d6f4fa5a5c0d1d3ae300b4c4d3b7d 100644 --- a/release-assistant/javcra/application/modifypart/modifyentrance.py +++ b/release-assistant/javcra/application/modifypart/modifyentrance.py @@ -16,6 +16,7 @@ Description: modify entrance import datetime import re import requests +from retrying import retry from javcra.api.gitee_api import Issue from javcra.common.constant import REPO_BASE_URL from javcra.libs.log import logger @@ -509,16 +510,16 @@ class Operation(Issue): "not allowed 'operate' value,expected in ['init','add','delete','update'],but given {}".format(operate) ) - def init(self): + def init(self, *args): """ init specific block Returns: init str """ - return self.get_new_issue_body(operate="init") + return self.get_new_issue_body(operate="init", *args) - def get_new_issue_body(self, operate="init", body_str=None, issues=None): + def get_new_issue_body(self, *args, operate="init", body_str=None, issues=None): raise NotImplementedError @@ -548,30 +549,49 @@ class CveIssue(Operation): return True logger.error("The CVE List file fails to be archived %s" % response.status_code) return False - except requests.RequestException as error: + except (requests.RequestException, AttributeError) as error: logger.error("The CVE List file fails to be archived because %s " % error) return False - def get_cve_list(self): + def get_cve_list(self, *args): """ Obtain cVE-related information provided by the CVE-Manager. Returns: cve_list: Data in Excel in dictionary form """ + useremail, obs_ak, obs_sk = args + + @retry(stop_max_attempt_number=3, wait_fixed=1000) + def create_list(): + + res = self.create_cve_list(useremail) + if not res: + raise ValueError("cve_manage archiving failed") + + @retry(stop_max_attempt_number=5, wait_fixed=60000) + def get_list(): + now_time = datetime.date( + datetime.date.today().year, + datetime.date.today().month, + datetime.date.today().day, + ).strftime("%Y-%m-%d") + branch_name = self.get_update_issue_branch() + if not branch_name: + logger.error("Failed to obtain branch") + return [] + cve_list = download_file( + now_time, "{}_updateinfo.xlsx".format(branch_name), obs_ak, obs_sk + ) + if not cve_list: + logger.error("Failed to obtain CVE data") + raise ValueError("Failed to obtain CVE data") + return cve_list - now_time = datetime.date(datetime.date.today().year, datetime.date.today().month, - datetime.date.today().day).strftime('%Y-%m-%d') - branch_name = self.get_update_issue_branch() - if not branch_name: - logger.error("Failed to obtain branch") - return [] - cve_list = download_file(now_time, "{}_updateinfo.xlsx".format(branch_name)) - if not cve_list: - logger.error("Failed to obtain CVE data") - return [] + create_list() + cve_list = get_list() return cve_list - def get_new_issue_body(self, operate="init", body_str=None, issues=None): + def get_new_issue_body(self, *args, operate="init", body_str=None, issues=None): """ get new issue body for cve block operation @@ -588,7 +608,7 @@ class CveIssue(Operation): t_head = ["CVE", "仓库", "status", "score", "version", "abi是否变化"] block_name = "## 1、CVE" - cve_list = [] if operate != "init" else self.get_cve_list() + cve_list = [] if operate != "init" else self.get_cve_list(*args) cve_prefix = "修复CVE {}个".format(len(cve_list)) return self.operate_for_specific_block(t_head, block_name, prefix=cve_prefix, operate=operate, @@ -599,7 +619,7 @@ class BugFixIssue(Operation): def __init__(self, repo, token, issue_num): super().__init__(repo, token, issue_num) - def get_new_issue_body(self, operate="init", body_str=None, issues=None): + def get_new_issue_body(self, *args, operate="init", body_str=None, issues=None): """ get new issue body for bugfix block operation @@ -646,7 +666,7 @@ class RequiresIssue(Operation): # so it is assumed that the return value is [] return [] - def get_new_issue_body(self, operate="init", body_str=None, issues=None): + def get_new_issue_body(self, *args, operate="init", body_str=None, issues=None): """ get new issue body for requires block operation @@ -674,7 +694,7 @@ class InstallBuildIssue(Operation): def __init__(self, repo, token, issue_num): super().__init__(repo, token, issue_num) - def get_new_issue_body(self, operate="init", body_str=None, issues=None): + def get_new_issue_body(self, *args, operate="init", body_str=None, issues=None): """ get new issue body for install build block operation @@ -702,7 +722,7 @@ class RemainIssue(Operation): def __init__(self, repo, token, issue_num): super().__init__(repo, token, issue_num) - def get_new_issue_body(self, operate="init", body_str=None, issues=None): + def get_new_issue_body(self, *args, operate="init", body_str=None, issues=None): """ get new issue body for remain block operation @@ -967,7 +987,7 @@ class IssueOperation(Operation): return False return True - def init_issue_description(self): + def init_issue_description(self, *args): """ initialize the release issue body when commenting "start-update" command @@ -979,7 +999,7 @@ class IssueOperation(Operation): return False release_range = "# 1、发布范围\n" - cve_block_str = self.cve_object.init() + cve_block_str = self.cve_object.init(*args) bugfix_block_str = self.bugfix_object.init() requires_block_str = self.requires_object.init() repo_block_str = self.init_repo_table() @@ -1030,7 +1050,7 @@ class IssueOperation(Operation): return True if self.update_issue(body=body_str) else False - def operate_release_issue(self, operation="init", operate_block=None, issues=None): + def operate_release_issue(self, *args, operation="init", operate_block=None, issues=None): """ modify entrance of the release issue @@ -1045,7 +1065,7 @@ class IssueOperation(Operation): """ try: if operation == "init": - return self.init_issue_description() + return self.init_issue_description(*args) else: return self.update_issue_description( operate=operation, update_block=operate_block, issues=issues diff --git a/release-assistant/javcra/cli/base.py b/release-assistant/javcra/cli/base.py index df8ec6103f01dfad4e3fcc9ef2549708a8145ee6..102da532ce024050b425afe58bc4195147152452 100644 --- a/release-assistant/javcra/cli/base.py +++ b/release-assistant/javcra/cli/base.py @@ -19,6 +19,7 @@ import argparse from collections import namedtuple from abc import abstractmethod + class BaseCommand(): """ Description: Basic attributes used for command invocation @@ -45,11 +46,11 @@ class BaseCommand(): self.sub_args = namedtuple( 'sub_args', ['sub_command', 'help', 'default', 'action', 'nargs', 'required', 'choices'] - ) + ) - def add_subcommand_with_2_args(self, sub_command, help_desc): + def add_subcommand_with_three_args(self, sub_command, help_desc): """ - Description: add subcommand with releaseIssueID and gitee ID as sub_parse argument + Description: add subcommand with releaseIssueID and gitee ID and token as sub_parse argument Args: Returns: @@ -71,39 +72,38 @@ class BaseCommand(): action='store', required=True ) + self.sub_parse.add_argument( + "--token", + type=str, + help="a valid GiteeToken value", + action="store", + required=True, + ) - def add_issueid_arg(self): + def add_obs_ak_arg(self): """ - Description: add release issud ID into sub_parse argument - Args: - - Returns: - - Raises: - + Description: add obs ak into sub_parse argument """ + self.sub_parse.add_argument( - 'releaseIssueID', - help='start release issue ID', - action='store' + "--ak", + type=str, + help="provide your access key", + action="store", + required=True, ) - def add_giteeid_arg(self): + def add_obs_sk_arg(self): """ - Description: add gitee ID into sub_parse argument - Args: - - Returns: - - Raises: - + Description: add obs ak into sub_parse argument """ self.sub_parse.add_argument( - '--giteeid', - help='the Gitee ID who trigger this command', - action='store', - required=True + "--sk", + type=str, + help="provide your secret key", + action="store", + required=True, ) @staticmethod diff --git a/release-assistant/javcra/cli/commands/__init__.py b/release-assistant/javcra/cli/commands/__init__.py index 6851032853d1af6ec5065266cf9d86c2bd1c0b3c..5773df8d03fbc556381473fb961af8d38f64dfa4 100644 --- a/release-assistant/javcra/cli/commands/__init__.py +++ b/release-assistant/javcra/cli/commands/__init__.py @@ -10,3 +10,47 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ +from javcra.application.checkpart.checktest import CheckTest +from javcra.application.serialize.validate import validate, validate_giteeid +from javcra.common.constant import GITEE_REPO + + +def personnel_authority(param_dict): + """ + personnel name and responsibilities acquisition + Args: + param_dict: parameter dictionary + + Returns: + personnel_dict: personnel Information Dictionary + """ + check = CheckTest(GITEE_REPO, param_dict.get("token"), param_dict.get("issueid")) + personnel_dict = check.parsing_body() + if not personnel_dict: + print("[ERROR] Failed to get the list of personnel permissions") + return {} + return personnel_dict + + +def parameter_permission_validate(schema, param_dict, comment): + """ + parameter verification and authorization verification + Args: + schema: validator class + param_dict: parameter dictionary + comment: comment + Returns: + return true if the verification is passed, otherwise return false + """ + _, error = validate(schema, param_dict, load=True) + if error: + print("Parameter validation failed") + return False + personnel_dict = personnel_authority(param_dict) + if not personnel_dict: + return False + permission = validate_giteeid(param_dict.get("giteeid"), comment, personnel_dict) + if not permission: + print("[ERROR] The current user does not have relevant operation permissions") + return False + return True diff --git a/release-assistant/javcra/cli/commands/checkpart.py b/release-assistant/javcra/cli/commands/checkpart.py index 38e2204c17dc2e5520062e4f5e4a833cc24aa165..4a7eb25e454cfa56640ed09d57aeb702b0807805 100644 --- a/release-assistant/javcra/cli/commands/checkpart.py +++ b/release-assistant/javcra/cli/commands/checkpart.py @@ -17,9 +17,9 @@ Class:CheckCommand from javcra.cli.base import BaseCommand from javcra.application.serialize.validate import validate_giteeid -from javcra.application.checkpart.checkentrance import CheckEntrance from javcra.common.constant import PERMISSION_DICT + class CheckCommand(BaseCommand): """ Description: start the check part @@ -33,8 +33,8 @@ class CheckCommand(BaseCommand): Description: Instance initialization """ super(CheckCommand, self).__init__() - self.add_subcommand_with_2_args(sub_command='check', - help_desc="release assistant of check part") + self.add_subcommand_with_three_args('check', + help_desc="release assistant of check part") self.sub_parse.add_argument( '--type', help='the type of check part, \ @@ -43,7 +43,7 @@ class CheckCommand(BaseCommand): nargs=None, required=True, choices=['cve', 'bug', 'status', 'requires', 'test'] - ) + ) self.sub_parse.add_argument( '--result', @@ -53,7 +53,7 @@ class CheckCommand(BaseCommand): nargs=None, required=False, choices=['yes', 'no'] - ) + ) def do_command(self, params): """ @@ -71,15 +71,3 @@ class CheckCommand(BaseCommand): permission = validate_giteeid(issue_id, gitee_id, PERMISSION_DICT.get(params.type)) if not permission: return - - type_status = params.result - type_dict = { - 'cve': CheckEntrance(issue_id, type_status).check_pkglist_result, - 'bug': CheckEntrance(issue_id, type_status).check_pkglist_result, - 'status': CheckEntrance(issue_id, type_status).check_issue_status, - 'requires': CheckEntrance(issue_id, type_status).check_requires, - 'test': CheckEntrance(issue_id, type_status).check_test_result - } - - print("check part start", issue_id, gitee_id) - type_dict.get(params.type)() diff --git a/release-assistant/javcra/cli/commands/modifypart.py b/release-assistant/javcra/cli/commands/modifypart.py index afecd30bfddcae40ddf92f4d52310af3878c70bc..cd4714b8815501cdd7a50c498f25e19c22c2927f 100644 --- a/release-assistant/javcra/cli/commands/modifypart.py +++ b/release-assistant/javcra/cli/commands/modifypart.py @@ -16,9 +16,11 @@ Class:ModifyCommand """ from javcra.cli.base import BaseCommand -from javcra.application.serialize.validate import validate_giteeid -from javcra.application.modifypart.modifyentrance import ModifyEntrance -from javcra.common.constant import PERMISSION_DICT +from javcra.cli.commands import parameter_permission_validate +from javcra.application.modifypart.modifyentrance import IssueOperation +from javcra.application.serialize.serialize import ModifySchema +from javcra.common.constant import GITEE_REPO + class ModifyCommand(BaseCommand): """ @@ -33,26 +35,26 @@ class ModifyCommand(BaseCommand): Description: Instance initialization """ super(ModifyCommand, self).__init__() - self.add_subcommand_with_2_args(sub_command='modify', - help_desc="release assistant of modify part") + self.add_subcommand_with_three_args('modify', + help_desc="release assistant of modify part") group = self.sub_parse.add_mutually_exclusive_group(required=True) group.add_argument( '--add', - help='adding a list of issues which would add to cve, bugfix, test or release list', + help='adding a list of issues which would add to cve, bugfix, test or remain list', default='', action='store', - choices=['cve', 'bug', 'release'] + choices=['cve', 'bugfix', 'remain'] ) group.add_argument( '--delete', - help='deleting a list of issues which would add to cve, bugfix, test or release list', + help='deleting a list of issues which would add to cve, bugfix, test or remain list', default='', action='store', - choices=['cve', 'bug', 'release'] + choices=['cve', 'bugfix', 'remain'] ) self.sub_parse.add_argument( '--id', - help='the list of issueid which would add/del to cve, bugfix, test or release list', + help='the list of issueid which would add/del to cve, bugfix, test or remain list', action='store', nargs='*', required=True, @@ -68,30 +70,41 @@ class ModifyCommand(BaseCommand): Raises: """ - issue_id = params.releaseIssueID - gitee_id = params.giteeid - issue_list = params.id - modify_type = '' - - if params.add: - modify_type = params.add - action = 'add' - elif params.delete: - modify_type = params.delete - action = 'del' - else: - print("Cannot parse the parameter") - return + # parameter dictionary + param_dict = { + "issueid": params.releaseIssueID, + "giteeid": params.giteeid, + "id": params.id, + "choice": params.add or params.delete, + "token": params.token, + } - permission = validate_giteeid(issue_id, gitee_id, PERMISSION_DICT.get(modify_type)) - if not permission: + # get action and modify_type according to parameters + action = "add" if params.add else "delete" + modify_type = params.add if params.add else params.delete + comment = "/" + action + "-" + modify_type + if modify_type == "remain": + comment = "/no-release" + validate_result = parameter_permission_validate( + ModifySchema, param_dict, comment + ) + if not validate_result: return - type_dict = { - 'cve': ModifyEntrance(issue_id, issue_list).modify_cve_list, - 'bug': ModifyEntrance(issue_id, issue_list).modify_bugfix_list, - 'release': ModifyEntrance(issue_id, issue_list).modify_release_result - } - - print("modify part start!", issue_id, gitee_id, issue_list) - type_dict.get(modify_type)(action) + # add or delete issues according to your choice + issue = IssueOperation(GITEE_REPO, params.token, params.releaseIssueID) + modify_res = issue.operate_release_issue(operation=action, operate_block=modify_type, issues=params.id) + if modify_res: + print("[INFO] %s %s in %s successfully." % (action, ",".join(params.id), modify_type)) + else: + print( + "[ERROR] failed to %s %s in %s." + % (action, ",".join(params.id), modify_type) + ) + if modify_type == "remain": + check_res = issue.update_remain_issue_state(params.id, action) + if check_res: + print("update remain issues successfully.") + else: + print("failed to update remain issues, " + "please check whether the issue exist in cve and bugfix part.") diff --git a/release-assistant/javcra/cli/commands/releasepart.py b/release-assistant/javcra/cli/commands/releasepart.py index a644601cb38a8db4455f3ca4b7851201685d4f5f..665d149225f042cfab67ee610ce7d0156f165efa 100644 --- a/release-assistant/javcra/cli/commands/releasepart.py +++ b/release-assistant/javcra/cli/commands/releasepart.py @@ -20,6 +20,7 @@ from javcra.application.serialize.validate import validate_giteeid from javcra.application.releasepart.releaseentrance import ReleaseEntrance from javcra.common.constant import PERMISSION_DICT + class ReleaseCommand(BaseCommand): """ Description: start the release part @@ -33,8 +34,7 @@ class ReleaseCommand(BaseCommand): Description: Instance initialization """ super(ReleaseCommand, self).__init__() - self.add_subcommand_with_2_args(sub_command='release', - help_desc="release assistant of release part") + self.add_subcommand_with_three_args('release', help_desc="release assistant of release part") self.sub_parse.add_argument( '--type', help='Specify the release check type, only allow checkok and cvrfok', diff --git a/release-assistant/javcra/cli/commands/startpart.py b/release-assistant/javcra/cli/commands/startpart.py index 02b67e609da7c3eacaa3cd15811f840b83cf3c24..ea6e82ace52733c894480f3cdad38aabffe9fd67 100644 --- a/release-assistant/javcra/cli/commands/startpart.py +++ b/release-assistant/javcra/cli/commands/startpart.py @@ -15,10 +15,12 @@ Description: start method's entrance for custom commands Class:StartCommand """ +from javcra.application.modifypart.modifyentrance import IssueOperation +from javcra.application.serialize.serialize import StartSchema from javcra.cli.base import BaseCommand -from javcra.application.serialize.validate import validate_giteeid -from javcra.application.startpart.startentrance import StartEntrance -from javcra.common.constant import PERMISSION_DICT +from javcra.cli.commands import parameter_permission_validate +from javcra.common.constant import GITEE_REPO + class StartCommand(BaseCommand): """ @@ -33,8 +35,17 @@ class StartCommand(BaseCommand): Description: Instance initialization """ super(StartCommand, self).__init__() - self.add_subcommand_with_2_args(sub_command='start', - help_desc="release assistant of start part") + self.add_subcommand_with_three_args('start', + help_desc="release assistant of start part") + self.add_obs_ak_arg() + self.add_obs_sk_arg() + self.sub_parse.add_argument( + "--useremail", + type=str, + help="the user's email address", + action="store", + required=True, + ) def do_command(self, params): """ @@ -46,12 +57,25 @@ class StartCommand(BaseCommand): Raises: """ - issue_id = params.releaseIssueID - gitee_id = params.giteeid - - permission = validate_giteeid(issue_id, gitee_id, PERMISSION_DICT.get('start')) - if not permission: + comment = "/start-update" + param_dict = { + "issueid": params.releaseIssueID, + "giteeid": params.giteeid, + "repo": GITEE_REPO, + "useremail": params.useremail, + "ak": params.ak, + "sk": params.sk, + "token": params.token, + } + validate_result = parameter_permission_validate( + StartSchema, param_dict, comment + ) + if not validate_result: return - - print("start part", issue_id, gitee_id) - StartEntrance().get_pkg_list() + issue = IssueOperation(GITEE_REPO, params.token, params.releaseIssueID) + args = (params.useremail, params.ak, params.sk) + start_res = issue.operate_release_issue(operation="init", *args) + if start_res: + print("[INFO] start update successfully.") + else: + print("[ERROR] failed to start update.") diff --git a/release-assistant/javcra/common/constant.py b/release-assistant/javcra/common/constant.py index 816846387707a7d6b96a9563bc54e3d5c0975803..5830222fed363dec415bf1b7782049252d38e299 100644 --- a/release-assistant/javcra/common/constant.py +++ b/release-assistant/javcra/common/constant.py @@ -69,6 +69,7 @@ AARCH_FRAME = "aarch64" X86_FRAME = "x86_64" + # branch list for standard epol list BRANCH_LIST = ["openEuler-20.03-LTS-SP1", "openEuler-20.03-LTS-SP2", "openEuler-20.03-LTS"] @@ -87,3 +88,6 @@ EPOL_SRC_NAME = ["published-Epol-src"] # repo base url REPO_BASE_URL = "http://121.36.84.172/repo.openeuler.org/" +# git warehouse address +GITEE_REPO = "release-management" + diff --git a/release-assistant/javcra/libs/read_excel.py b/release-assistant/javcra/libs/read_excel.py index 07edb6f3361f304c600941e82317327ecea03e03..cd0152867d2800d553a086efaacbd12a4c326a0e 100644 --- a/release-assistant/javcra/libs/read_excel.py +++ b/release-assistant/javcra/libs/read_excel.py @@ -60,19 +60,21 @@ def read_excel_xlsx(curr_path): return out_list -def download_file(now_time, file_name): +def download_file(now_time, file_name, obs_ak, obs_sk): """ download file content Args: now_time: now_time file_name: file_name + obs_ak: obs ak + obs_sk: obs sk Returns: cve_list: """ file_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) temp_path = os.path.join(file_path, "tmp{}".format(str(uuid.uuid1().hex))) try: - obs_client = ObsCloud(os.getenv("ak"), os.getenv("sk"), CVE_MANAGE_SERVER, CVE_MANAGE_BUCKET_NAME) + obs_client = ObsCloud(obs_ak, obs_sk, CVE_MANAGE_SERVER, CVE_MANAGE_BUCKET_NAME) # Determine whether the file to be downloaded is in the object of the bucket files = obs_client.bucket_list("{}/{}".format(CVE_UPDATE_INFO, now_time)) file_object = ""