From 6e7e1711d4ff63b0a73b72b19b20454407b52227 Mon Sep 17 00:00:00 2001 From: tushenmei Date: Tue, 13 Jul 2021 11:48:36 +0800 Subject: [PATCH] gitee api --- release-assistant/javcra/api/gitee_api.py | 317 +++++++++++++++++- .../javcra/libs/config/global_config.py | 4 +- release-assistant/javcra/libs/log.py | 68 +++- 3 files changed, 385 insertions(+), 4 deletions(-) diff --git a/release-assistant/javcra/api/gitee_api.py b/release-assistant/javcra/api/gitee_api.py index d4c26bb..79a9b3c 100644 --- a/release-assistant/javcra/api/gitee_api.py +++ b/release-assistant/javcra/api/gitee_api.py @@ -13,4 +13,319 @@ """ Description: conver multiple gitee restful APIs to python methods Class: -""" \ No newline at end of file +""" + +import json +import re +import time +import requests +from requests.exceptions import RequestException +from javcra.libs.log import logger + + +class Issue: + """cover gitee APIs to python methods""" + + def __init__(self, repo, token, issue_num): + self.headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) " + "Gecko/20100101 Firefox/50.0" + } + self.repo = repo + self.owner = "src-openeuler" + self.token = token + self.issue_num = issue_num + self.date = time.strftime("%Y%m%d", time.localtime()) + + def __generate_request_params(self, **kwargs): + """ + + generate request parameters + + Returns: + dict: requests params + """ + params = {"repo": self.repo, "access_token": self.token} + params.update(**kwargs) + return params + + def gitee_api_request(self, method, url, params=None, retry=3): + """ + requesting gitee api + + Args: + method : http request method + url : request url + params : request params + retry: retry count + + Returns: + resp: response of request + """ + if not params: + params = {} + success_code = {201, 200} + + try: + resp = requests.request(method.lower(), url, params=params, headers=self.headers) + + retry_count = 0 + while resp.status_code == 408 and retry_count < retry: + logger.error("api request timed out, retrying %s.", retry_count) + resp = requests.request(method.lower(), url, params=params, headers=self.headers) + retry_count += 1 + + if resp.status_code not in success_code: + return None + except RequestException as e: + logger.error("RequestException occurred. %s" % e) + return None + return resp + + def __get_gitee_api_url(self, url_name, **kwargs): + + url_prefix = "https://gitee.com/api/v5/" + + url_dict = { + "pkg_issues_url": url_prefix + "repos/{owner}/{repo}/issues?access_token={access_token}&state=open" + .format(owner=self.owner, repo=kwargs.get("pkg"), access_token=self.token), + + "issue_url": url_prefix + "enterprises/{enterprise}/issues/{number}?access_token={token}".format( + enterprise=kwargs.get("owner"), number=kwargs.get("issue_id"), token=self.token), + + "create_issue_url": url_prefix + "repos/{owner}/issues".format(owner=kwargs.get("owner")), + + "update_issue_url": url_prefix + "repos/{owner}/issues/{number}".format(owner=self.owner, + number=self.issue_num), + + "create_comment_url": url_prefix + "repos/{owner}/{repo}/issues/{number}/comments".format( + owner=self.owner, repo=self.repo, number=self.issue_num) + } + + return url_dict.get(url_name) + + def __get_pkg_issues(self, pkg): + """ + list all open issues of pkg + Args: + pkg: package name + + Returns: + issue list of specific pkg repository + """ + issue_url = self.__get_gitee_api_url("pkg_issues_url", pkg=pkg) + resp = self.gitee_api_request("get", issue_url) + if resp: + issues = json.loads(resp.text) + return issues + else: + logger.error("failed to get the issue info of pkg %s." % pkg) + return [] + + def get_issue_info(self, issue_number, owner="open_euler"): + """ + get info of specified issue in enterprise repository + + Args: + issue_number: issue id + owner: enterprise repository + + Returns: + + """ + issue_url = self.__get_gitee_api_url("issue_url", owner=owner, issue_id=issue_number) + resp = self.gitee_api_request("get", issue_url) + if resp: + return json.loads(resp.text) + + logger.error("failed to get the issue info of %s. " % issue_number) + return None + + def create_issue(self, params): + """ + create issue in gitee + Args: + params: parameters to create issue + + Returns: + created_issue_id + """ + # get all the open state issue's title + exist_issues = self.__get_pkg_issues(params.get("repo")) + exist_issue_title_dict = dict() + for issue in exist_issues: + issue_id = issue.get("number") + issue_title = issue.get("title") + exist_issue_title_dict[issue_id] = issue_title + + issue_title = params.get("title") + + # if already exist the same title issue, then return the existed issue id + if issue_title in exist_issue_title_dict.values(): + for created_issue_id, title in exist_issue_title_dict.items(): + if title == issue_title: + logger.info("already exists the issue: {}, issue number is {}.".format( + issue_title, created_issue_id)) + return created_issue_id + else: + prj_issue_url = self.__get_gitee_api_url("create_issue_url", owner=params["owner"]) + post_res = self.gitee_api_request("post", prj_issue_url, params) + if not post_res: + logger.error("failed to create the issue: {}".format(issue_title)) + return None + + resp_content = json.loads(post_res.text) + created_issue_id = resp_content["number"] + return created_issue_id + + def update_issue(self, **kwargs): + """ + call the gitee api to update the issue for release issue + + Returns: + update issue result + """ + params = self.__generate_request_params(**kwargs) + update_issue_url = self.__get_gitee_api_url("update_issue_url") + + return self.gitee_api_request( + "patch", + url=update_issue_url, + params=params + ) + + def create_issue_comment(self, comment): + """ + method to create issue comment for release issue + + Args: + comment (str): comment str + """ + url = self.__get_gitee_api_url("create_comment_url") + params = {"body": comment, "access_token": self.token} + resp = self.gitee_api_request("post", url=url, params=params) + return resp + + def get_update_issue_branch(self): + """ + get branch from issue title for making update repo + + Returns: + update issue branch + + """ + branch = None + issue_content = self.get_issue_info(self.issue_num) + try: + if issue_content: + title = issue_content["title"] + + # verify weather the title is correct, title demo: openxxxx-20.03-LTS-SP1 Update 2021/4/23 release + title_content = title.strip().split() + if len(title_content) != 4: + logger.error("incorrect title, please check. Title: %s" % title) + return None + branch = title.strip().split()[0] + except IndexError: + logger.error("failed to get update issue branch.") + return branch + + def get_issue_body(self, issue_id): + """ + get issue body content for release issue + + Returns: + str: issue body str + """ + issue_content = self.get_issue_info(issue_id) + if not issue_content: + logger.error("can not get issue content for %s ,please check!" % issue_id) + return None + body = issue_content.get("body", "") + if not body: + logger.error(" %s empty issue body,please check." % issue_id) + return body + + @staticmethod + def __process_body_for_pkglist(body, block_name): + """ + process the md string of cve bugfix requires for pkg_list + + Args: + body: block body + block_name: given [cve,bugfix,req] + + Returns: + current pkgs in block + """ + # using regular to match the specific issue content in the table + content = re.compile("\|\-\|\\n(?P.*)", re.S).search(body) + if not content: + content = "" + else: + content = content["content"] + + repos = set() + # If the part is cve or bugfix, then the second column is pkgname + pkgname_index_dict = {"cve": 2, "bugfix": 2, "req": 1} + + for line in content.splitlines(): + if not line: + continue + try: + repo_idx = pkgname_index_dict.get(block_name) + repo_info = line.split("|")[repo_idx] + except IndexError: + logger.error("can not get pkg info in {} block".format(block_name)) + else: + repos.add(repo_info) + return repos + + def __get_pkglist_from_specific_part(self, block_res, block_name, pkg_set): + """ + get pkglist from cve part、bugfix part、requires part + + Args: + block_res: cve part res、bugfix part res、requires part res + block_name: cve、bugfix、requires + pkg_set: set of pkglist + + Returns: + pkg_set + """ + # If exists specific part, then get packages + if block_res: + pkg_set.update(self.__process_body_for_pkglist(block_res[block_name], block_name=block_name)) + else: + logger.warning("not found %s content when getting pkglist from specific part.", block_name) + return pkg_set + + def get_update_list(self): + """ + use regular to get pkglist + + Args: + issue_body: issue body str + + Returns: + list: pkg list + """ + issue_body = self.get_issue_body(self.issue_num) + + if issue_body: + pkgs = set() + + # using regular to get the contents of the specified part + cve_res = re.compile("(?P1、CVE.*?\\n\\n)", re.S).search(issue_body) + bugfix_res = re.compile("(?P2、bugfix.*?\\n\\n)", re.S).search(issue_body) + req_res = re.compile("(?P3、requires.*?\\n\\n)", re.S).search(issue_body) + + # get packages from cve、bugfix、requires table + cve_pkgs = self.__get_pkglist_from_specific_part(cve_res, "cve", pkgs) + cve_bugfix_pkgs = self.__get_pkglist_from_specific_part(bugfix_res, "bugfix", cve_pkgs) + cve_bugfix_req_pkgs = self.__get_pkglist_from_specific_part(req_res, "req", cve_bugfix_pkgs) + + return list(cve_bugfix_req_pkgs) + else: + logger.error("empty content of issue body, can not get update list.") + return [] diff --git a/release-assistant/javcra/libs/config/global_config.py b/release-assistant/javcra/libs/config/global_config.py index 89b2f73..a358d75 100644 --- a/release-assistant/javcra/libs/config/global_config.py +++ b/release-assistant/javcra/libs/config/global_config.py @@ -51,4 +51,6 @@ OPENEULER_BRANCH_MAP = os.path.join( "check_requires", "support_branch.yaml" ) -SUPPORTED_BRANCHES_LIST = ["openEuler"] \ No newline at end of file +SUPPORTED_BRANCHES_LIST = ["openEuler"] + +LOG_DIR = "./" \ No newline at end of file diff --git a/release-assistant/javcra/libs/log.py b/release-assistant/javcra/libs/log.py index 465d175..ae2005e 100644 --- a/release-assistant/javcra/libs/log.py +++ b/release-assistant/javcra/libs/log.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +# !/usr/bin/python3 # ****************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # licensed under the Mulan PSL v2. @@ -12,4 +12,68 @@ # ******************************************************************************/ """ Logging related -""" \ No newline at end of file +""" +import logging +import os +import pathlib +from concurrent_log_handler import ConcurrentRotatingFileHandler +from javcra.libs.config.global_config import LOG_DIR + + +class Log(object): + """ + operation log of the system + """ + + def __init__(self, name=__name__, path=None): + self.__current_rotating_file_handler = None + if not path: + path = os.path.dirname(__file__) + self.__path = os.path.join(path, "log_info.log") + + if not os.path.exists(self.__path): + try: + os.makedirs(os.path.split(self.__path)[0]) + except FileExistsError: + pathlib.Path(self.__path).touch(mode=0o644) + self.__max_bytes = 30000000 + self.__backup_count = 2 + self.__level = "INFO" + self.__logger = logging.getLogger(name) + self.__logger.setLevel(self.__level) + + def __init_handler(self): + self.__current_rotating_file_handler = ConcurrentRotatingFileHandler( + filename=self.__path, + mode="a", + maxBytes=self.__max_bytes, + backupCount=self.__backup_count, + encoding="utf-8", + use_gzip=True, + ) + self.__set_formatter() + self.__set_handler() + + def __set_formatter(self): + formatter = logging.Formatter( + "%(asctime)s-%(filename)s-[line:%(lineno)d]" + "-%(levelname)s-[ log details ]: %(message)s", + datefmt="%a, %d %b %Y %H:%M:%S", + ) + self.__current_rotating_file_handler.setFormatter(formatter) + + def __set_handler(self): + self.__current_rotating_file_handler.setLevel(self.__level) + self.__logger.addHandler(self.__current_rotating_file_handler) + + @property + def logger(self): + """ + Gets the logger property + """ + if not self.__current_rotating_file_handler: + self.__init_handler() + return self.__logger + + +logger = Log(__name__, path=LOG_DIR).logger -- Gitee