diff --git a/release-assistant/javcra/api/gitee_api.py b/release-assistant/javcra/api/gitee_api.py index 062ae2320a8b1ae29c92c0ac431432cfe926cf64..3d4db794bf1cf855668b0484c9ccfc7c4fe53139 100644 --- a/release-assistant/javcra/api/gitee_api.py +++ b/release-assistant/javcra/api/gitee_api.py @@ -231,6 +231,9 @@ class Issue: logger.error("incorrect title, please check. Title: %s" % title) return None branch = title.strip().split()[0] + if branch not in BRANCH_LIST: + logger.error("branch %s not in qualified branch_list: %s" % (branch, BRANCH_LIST)) + return None except IndexError: logger.error("failed to get update issue branch.") return branch diff --git a/release-assistant/javcra/api/jenkins_api.py b/release-assistant/javcra/api/jenkins_api.py index 3e56684b2aceb1edc76f22dbbc8e378a992961e0..1e89ec23aed25a5457a6095a450af76a3c4df3f5 100644 --- a/release-assistant/javcra/api/jenkins_api.py +++ b/release-assistant/javcra/api/jenkins_api.py @@ -15,6 +15,7 @@ Description: conver multiple Jenkins restful APIs to python methods Class: """ import math +import re import time import xml.etree.ElementTree as ET from collections import defaultdict @@ -549,11 +550,12 @@ class JenkinsJob(object): if "Starting building" in line: line_info = line.split() _job_name = line_info[2] - _build_id = line_info[3].strip("#") - job_name_id_dict[_job_name] = _build_id + build_id_info = line_info[3] + job_name_id_dict[_job_name] = re.findall(r'\d+', build_id_info)[0] logger.info("finished to get build id dict: %s" % job_name_id_dict) except IndexError as err: logger.error("error in get self_build parallel job id. %s" % err) + return {} return job_name_id_dict diff --git a/release-assistant/javcra/application/serialize/serialize.py b/release-assistant/javcra/application/serialize/serialize.py index c3b7331c515c1627a9e5949a219ba6401fdeccef..51c1c737bff922b36e46c64b59e7d42dc0b45558 100644 --- a/release-assistant/javcra/application/serialize/serialize.py +++ b/release-assistant/javcra/application/serialize/serialize.py @@ -67,3 +67,5 @@ class ReleaseSchema(BaseSchema): type = fields.String(required=True, validate=validate.OneOf(["checkok", "cvrfok"])) jenkinsuser = fields.String(required=True, validate=validate.Length(min=1)) jenkinskey = fields.String(required=True, validate=validate.Length(min=1)) + publishuser = fields.String(required=True, validate=validate.Length(min=1)) + publishkey = fields.String(required=True, validate=validate.Length(min=1)) diff --git a/release-assistant/javcra/cli/base.py b/release-assistant/javcra/cli/base.py index e8a729516189541ee6c14c001436b82346edb09c..c5b351077bc7ac387d66979b30c388116aa8c27e 100644 --- a/release-assistant/javcra/cli/base.py +++ b/release-assistant/javcra/cli/base.py @@ -106,6 +106,31 @@ class BaseCommand(): required=True, ) + @staticmethod + def create_comment(type_res, result, issue): + """ + create comment for jenkins job + """ + if not result: + raise ValueError("%s: No comment information. The content is: %s." % (type_res, result)) + + comment_res = issue.create_jenkins_comment(result) + if not comment_res: + print("[ERROR] failed to create %s" % type_res) + else: + print("[INFO] successfully create %s" % type_res) + + @staticmethod + def get_branch_pkgs(issue): + branch_name = issue.get_update_issue_branch() + if not branch_name: + raise ValueError("failed to get branch name.") + + update_pkgs = issue.get_update_list() + if not update_pkgs: + raise ValueError("failed to get obs_pkgs.") + return branch_name, update_pkgs + @staticmethod def register_command(command): """ diff --git a/release-assistant/javcra/cli/commands/checkpart.py b/release-assistant/javcra/cli/commands/checkpart.py index 227ef6e510ef3c7e69ffee1be83635a78bc8c99b..79f182e268b3e0b46ce54e6cc6a6f8df9bd36116 100644 --- a/release-assistant/javcra/cli/commands/checkpart.py +++ b/release-assistant/javcra/cli/commands/checkpart.py @@ -15,17 +15,28 @@ Description: check method's entrance for custom commands Class:CheckCommand """ +from javcra.api.jenkins_api import JenkinsJob +from javcra.application.checkpart.checkentrance import CheckEntrance +from javcra.application.checkpart.checktest import CheckTest +from javcra.application.modifypart.modifyentrance import IssueOperation +from javcra.application.serialize.serialize import CheckSchema from javcra.cli.base import BaseCommand -from javcra.application.serialize.validate import validate_giteeid -from javcra.common.constant import PERMISSION_DICT + +from javcra.cli.commands import parameter_permission_validate +from javcra.common import constant +from javcra.common.constant import ( + GITEE_REPO, + MAX_PARAL_NUM, + REALSE_TOOLS_BUCKET_NAME, + REALSE_TOOLS_SERVER, + X86_FRAME, CHECK_COMMENT_DICT) + +from javcra.api.obscloud import ObsCloud class CheckCommand(BaseCommand): """ Description: start the check part - Attributes: - sub_parse: Subcommand parameters - params: Command line parameters """ def __init__(self): @@ -33,41 +44,308 @@ class CheckCommand(BaseCommand): Description: Instance initialization """ super(CheckCommand, self).__init__() - self.add_subcommand_communal_args('check', - help_desc="release assistant of check part") + self.add_subcommand_communal_args('check', help_desc="release assistant of check part") self.sub_parse.add_argument( - '--type', - help='the type of check part, \ - including cve, bugfix, requires, issue status and test result', - action='store', + "--type", + help="the type of check part, \ + including requires, issue status and test result", + action="store", nargs=None, required=True, - choices=['cve', 'bug', 'status', 'requires', 'test'] + choices=["status", "requires", "test"], ) - self.sub_parse.add_argument( - '--result', - help='the check result, it would be yes or no', - default='yes', - action='store', - nargs=None, - required=False, - choices=['yes', 'no'] + "--ak", + type=str, + help="provide obs access key", + action="store", + required=True, + ) + self.sub_parse.add_argument( + "--sk", + type=str, + help="provide obs secret key", + action="store", + required=True, + ) + self.sub_parse.add_argument( + "--jenkinsuser", + type=str, + help="provide your jenkinsuser", + action="store", + required=True, + ) + self.sub_parse.add_argument( + "--jenkinskey", + type=str, + help="provide your jenkins key", + action="store", + required=True, ) - def do_command(self, params): + @staticmethod + def jenkins_server(params, paral_num, branch_name, release_date): """ - Description: Executing command + Description: to get the jenkins server + Args: + params: Command line parameters + paral_num: paral num of jenkins job + branch_name: branch name of release issue + release_date: date of release + + Returns: + jenkins server object + """ + + jenkins_server = JenkinsJob( + constant.JENKINS_BASE_URL, + params.jenkinsuser, + params.jenkinskey, + paral_num, + branch_name, + release_date, + ) + return jenkins_server + + @staticmethod + def check(params): + """ + Description: to get check object + Args: + params: Command line parameters + + Returns: + check object + """ + check = CheckTest(GITEE_REPO, params.token, params.releaseIssueID) + return check + + @staticmethod + def issue(params): + """ + Description: to get issue object + Args: + params: Command line parameters + + Returns: + issue object + """ + issue = IssueOperation(GITEE_REPO, params.token, params.releaseIssueID) + return issue + + @staticmethod + def check_issue(params): + """ + Description: to get check_issue object + Args: + params: Command line parameters + + Returns: + check_issue object + """ + check_issue = CheckEntrance(GITEE_REPO, params.token, params.releaseIssueID) + return check_issue + + def test_operation(self, params): + """ + Description: the operation for test Args: params: Command line parameters + Returns: + True or False + """ + review_res = self.check(params).people_review() + if not review_res: + print("[ERROR] failed to operate test in check part.") + return False + print("[INFO] successfully operate test in check part.") + return True + + def status_operation(self, params): + """ + Description: operation for status + Args: + params: Command line parameters - Raises: + Returns: + True or False + """ + status_res = self.issue(params).check_issue_state() + if not status_res: + print("[ERROR] failed to update status in check part.") + return False + print("[INFO] successfully update status in check part.") + return True + def requires_operation(self, params): """ - issue_id = params.releaseIssueID - gitee_id = params.giteeid + Description: operation for get requires + Args: + params: Command line parameters + + Returns: + """ + + def transfer_pkg_rpm(obs_project, pkg_family, pkgs): + """ + transfer the rpm package to the server, and comment on the Jenkins + job result on the release issue + """ + obs_job_params = { + "ScanOSSAPIURL": constant.JENKINS_API_URL, + "ScanOSSResultRepo": constant.JENKINS_SERVER_REPO, + "action": "create", + "obs_project": obs_project, + "update_dir": "update_" + issue.date, + "package_family": pkg_family, + "pkgnamelist": ",".join(pkgs), + } + jenkins_obs_res = jenkins_server.get_specific_job_comment( + obs_job_params, constant.OBS_RELEASE_JOB + ) + return jenkins_obs_res + + def verify_selfbuild(): + """ + verify selfbuild for update pkg_list + + """ + jenkins_server.delete_jenkins_job(constant.JENKINS_PATH_PREFIX) + created_res = jenkins_server.create_selfbuild_jenkins_jobs(update_pkgs) + if not created_res: + raise ValueError("failed to create selfbuild jenkins job.") + + build_status_res = jenkins_server.get_selfbuild_job_comment() + return build_status_res + + def verify_install(): + """ + verify install for pkgs list + """ + install_jobs_params = { + "ScanOSSAPIURL": constant.JENKINS_API_URL, + "ScanOSSResultRepo": constant.JENKINS_SERVER_REPO, + "ARCH": X86_FRAME, + "EPOL": epol_flag, + "UPDATE_TIME": issue.date, + "BRANCH": branch_name, + "PKGLIST": ",".join(update_pkgs), + } + install_status_res = jenkins_server.get_specific_job_comment( + install_jobs_params, constant.INSTALL_JOB_PREFIX + branch_name + ) + return install_status_res + + def issue_write_back(install_or_build, pkgs): + """ + create issue when install or selfbuild failed, and then write back to release issue + """ + install_or_build_dict = {"build_list": "build", "install_list": "install"} + for pkg in pkgs: + issue_id = issue.create_install_build_issue( + install_or_build_dict.get(install_or_build), pkg + ) + if not issue_id: + print("failed to create %s issue for %s." % (install_or_build, pkg)) + else: + write_res = issue.operate_release_issue( + operation="add", + operate_block="install_build", + issues=[issue_id], + ) + if not write_res: + print("failed to write back to install_build issue %s in release issue" % issue_id) + + issue = self.issue(params) + check_issue = self.check_issue(params) + branch_name, update_pkgs = self.get_branch_pkgs(issue) + + standard_list, epol_list = issue.get_standard_epol_list(branch_name, update_pkgs) + epol_flag = "True" if epol_list else "False" + + # get parallel jenkins job num according to length of pkg_list and max parallel num + paral_num = min(MAX_PARAL_NUM, len(update_pkgs)) - permission = validate_giteeid(issue_id, gitee_id, PERMISSION_DICT.get(params.type)) - if not permission: + # get jenkins_server and cloud server + jenkins_server = self.jenkins_server(params, paral_num, branch_name, issue.date) + cloud_server = ObsCloud( + params.ak, params.sk, REALSE_TOOLS_SERVER, REALSE_TOOLS_BUCKET_NAME + ) + + # delete install and build archive in hw cloud + res = cloud_server.delete_dir("install_build_log/{}".format(branch_name)) + if not res: + raise ValueError("obs cloud delete last archived file failed") + + # verify that the dependencies and write back the missing dependencies to the issue description + check_res = issue.operate_release_issue( + operation="add", operate_block="requires" + ) + if not check_res: + print("[ERROR] failed to get requires.") + else: + print("[INFO] successfully get requires.") + + # upload the rpm package to the server + obs_prj = branch_name.replace("-", ":") + stand_transfer_res = transfer_pkg_rpm(obs_prj, "standard", standard_list) + self.create_comment("transfer standard rpm jenkins res", stand_transfer_res, issue) + + if epol_list: + obs_prj = obs_prj + ":" + "Epol" + epol_transfer_res = transfer_pkg_rpm(obs_prj, "EPOL", epol_list) + self.create_comment("transfer epol rpm jenkins res", epol_transfer_res, issue) + + # add the repo content to the release issue + add_res = check_issue.add_repo_in_table() + if not add_res: + raise ValueError("failed to add repo in release issue.") + print("[INFO] successful to add repo in release issue.") + + # self-build verification + selfbuild_res = verify_selfbuild() + self.create_comment("selfbuild jenkins res", selfbuild_res, issue) + + # installation verification + install_res = verify_install() + self.create_comment("install jenkins res", install_res, issue) + + # obtain and analyze the self-build installation results from file server + parsed_install_build_res = cloud_server.parse_install_build_content(branch_name) + + # If the selfbuild or install fails, create issue and then write the issue_id back to the release issue + for issue_type, pkg_list in parsed_install_build_res.items(): + issue_write_back(issue_type, pkg_list) + + def do_command(self, params): + """ + Description: Executing command + Args: + params: Command line parameters + + """ + comment = CHECK_COMMENT_DICT.get(params.type) + if not comment: + print("[ERROR] not allowed operation, please check.") + return + + param_dict = { + "issueid": params.releaseIssueID, + "giteeid": params.giteeid, + "type": params.type, + "token": params.token, + "ak": params.ak, + "sk": params.sk, + "jenkinsuser": params.jenkinsuser, + "jenkinskey": params.jenkinskey, + } + validate_result = parameter_permission_validate( + CheckSchema, param_dict, comment + ) + if not validate_result: return + try: + getattr(self, "{}_operation".format(params.type))(params) + except ValueError: + print("not allowed operate type: %s in check part." % params.type) diff --git a/release-assistant/javcra/cli/commands/releasepart.py b/release-assistant/javcra/cli/commands/releasepart.py index dfdc59c2a94019aeaed43ef1011d68510b430e32..cf5e3176233513e6dae97d978aab71e7b47e2a62 100644 --- a/release-assistant/javcra/cli/commands/releasepart.py +++ b/release-assistant/javcra/cli/commands/releasepart.py @@ -15,10 +15,13 @@ Description: release method's entrance for custom commands Class:ReleaseCommand """ +from javcra.api.jenkins_api import JenkinsJob +from javcra.application.modifypart.modifyentrance import IssueOperation +from javcra.application.serialize.serialize import ReleaseSchema from javcra.cli.base import BaseCommand -from javcra.application.serialize.validate import validate_giteeid -from javcra.application.releasepart.releaseentrance import ReleaseEntrance -from javcra.common.constant import PERMISSION_DICT +from javcra.cli.commands import parameter_permission_validate +from javcra.common import constant +from javcra.common.constant import MAX_PARAL_NUM, GITEE_REPO class ReleaseCommand(BaseCommand): @@ -34,13 +37,93 @@ class ReleaseCommand(BaseCommand): Description: Instance initialization """ super(ReleaseCommand, self).__init__() - self.add_subcommand_communal_args('release', help_desc="release assistant of release part") + self.add_subcommand_communal_args('release', help_desc="release assistant of release part") self.sub_parse.add_argument( '--type', help='Specify the release check type, only allow checkok and cvrfok', action='store', choices=['checkok', 'cvrfok'] ) + self.sub_parse.add_argument( + "--jenkinsuser", + type=str, + help="provide your jenkinsuser", + action="store", + required=True, + ) + self.sub_parse.add_argument( + "--jenkinskey", + type=str, + help="provide your jenkins key", + action="store", + required=True, + ) + self.sub_parse.add_argument( + "--publishuser", + type=str, + help="provide your publishuser", + action="store", + required=True, + ) + self.sub_parse.add_argument( + "--publishkey", + type=str, + help="provide your publish key", + action="store", + required=True, + ) + + def checkok_operation(self, params): + """ + Description: operation for check ok + Args: + params: Command line parameters + Returns: + """ + + def publish_rpms(obs_project, pkg_family): + """ + publish the rpm package + """ + obs_job_params = { + "ScanOSSAPIURL": constant.JENKINS_API_URL, + "ScanOSSResultRepo": constant.JENKINS_SERVER_REPO, + "action": "release", + "obs_project": obs_project, + "update_dir": "update_" + issue.date, + "package_family": pkg_family, + } + jenkins_obs_res = jenkins_server.get_specific_job_comment( + obs_job_params, constant.OBS_RELEASE_JOB + ) + return jenkins_obs_res + + issue = IssueOperation(GITEE_REPO, params.token, params.releaseIssueID) + branch_name, update_pkgs = self.get_branch_pkgs(issue) + + # get parallel jenkins job num according to length of pkg_list and max parallel num + paral_num = min(MAX_PARAL_NUM, len(update_pkgs)) + + # get jenkins_server + jenkins_server = JenkinsJob( + constant.JENKINS_BASE_URL, + params.jenkinsuser, + params.jenkinskey, + paral_num, + branch_name, + issue.date, + ) + + # publish pkg rpms + obs_prj = branch_name.replace("-", ":") + standard_list, epol_list = issue.get_standard_epol_list(branch_name, update_pkgs) + stand_transfer_res = publish_rpms(obs_prj, "standard") + self.create_comment("transfer standard rpm jenkins res", stand_transfer_res, issue) + + if epol_list: + obs_prj = obs_prj + ":" + "Epol" + epol_transfer_res = publish_rpms(obs_prj, "EPOL") + self.create_comment("transfer epol rpm jenkins res", epol_transfer_res, issue) def do_command(self, params): """ @@ -49,14 +132,28 @@ class ReleaseCommand(BaseCommand): params: Command line parameters Returns: - Raises: - """ - issue_id = params.releaseIssueID - gitee_id = params.giteeid - permission = validate_giteeid(issue_id, gitee_id, PERMISSION_DICT.get(params.type)) - if not permission: + param_dict = { + "issueid": params.releaseIssueID, + "giteeid": params.giteeid, + "type": params.type, + "token": params.token, + "jenkinsuser": params.jenkinsuser, + "jenkinskey": params.jenkinskey, + "publishuser": params.publishuser, + "publishkey": params.publishkey + } + + release_type = params.type + comment = "/" + release_type[:-2] + "-" + release_type[-2:] + + validate_result = parameter_permission_validate( + ReleaseSchema, param_dict, comment + ) + if not validate_result: return - print("release part", issue_id, gitee_id) - ReleaseEntrance().release_check() + try: + getattr(self, "{}_operation".format(params.type))(params) + except ValueError: + print("not allowed operate type: %s in release part." % params.type) diff --git a/release-assistant/javcra/common/constant.py b/release-assistant/javcra/common/constant.py index 47695801759f858ecf7ce59c080ca0274406be12..6756e719ffaf18bb4b5cb38c8133df3a4b9df519 100644 --- a/release-assistant/javcra/common/constant.py +++ b/release-assistant/javcra/common/constant.py @@ -87,7 +87,6 @@ EPOL_SRC_NAME = ["published-Epol-src"] # repo base url REPO_BASE_URL = "http://121.36.84.172/repo.openeuler.org/" - # template job of aarch64 AARCH64_TM_JOB = "function-item/release-manager/update_template_jobs/aarch64/test_build" @@ -105,3 +104,35 @@ ACTUATOR_DICT = { # git warehouse address for release issue GITEE_REPO = "release-management" + +# ScanOSS API URL for jenkins trigger +JENKINS_API_URL = "https://shenzhen.scanoss.com/api/scan/direct" + +# server repo for jenkins trigger +JENKINS_SERVER_REPO = "121.36.53.23" + +# install jenkins job prefix +INSTALL_JOB_PREFIX = "function-item/release-manager/update_template_jobs/install_" + +# job for upload package rpm to server +OBS_RELEASE_JOB = "obs/update_release_pkg_rpm" + +# jenkins base url +JENKINS_BASE_URL = 'https://jenkins.openeuler.org' + +REALSE_TOOLS_BUCKET_NAME = "release-tools" + +REALSE_TOOLS_SERVER = "obs.cn-north-4.myhuaweicloud.com" + +# max parallel jenkins num +MAX_PARAL_NUM = 6 + +# obs_project +OBS_PRJ = "openEuler:20.09" + +# check comment dict +CHECK_COMMENT_DICT = { + "status": "/check-status", + "requires": "/get-requires", + "test": "/test-ok", +} \ No newline at end of file