From 788edf72a29a26ef9e770fc54083ed392ec55a01 Mon Sep 17 00:00:00 2001 From: small_leek Date: Tue, 29 Jun 2021 15:33:58 +0800 Subject: [PATCH 1/2] init tool for comparing version number differences fix bug Modify according to inspection comments Modify according to inspection comments --- release-assistant/javcra/api/versiondiff.py | 354 ++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 release-assistant/javcra/api/versiondiff.py diff --git a/release-assistant/javcra/api/versiondiff.py b/release-assistant/javcra/api/versiondiff.py new file mode 100644 index 0000000..95e7eea --- /dev/null +++ b/release-assistant/javcra/api/versiondiff.py @@ -0,0 +1,354 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-05-27 +# ******************************************************************************/ + +import argparse +import csv +import codecs + +from executecmd import ExecuteCmd +from file_manage import mv_files +from file_manage import copy_file +from loadconf import local_conf +import logging +import os +import re +import shutil +import smtplib +from sig_maintainer import MaintainerAndContributor +from itertools import islice +from email.mime.text import MIMEText +from email.header import Header +from pathlib import Path +from loadconf import LoadConfig + +class PkgVersionDiff(object): + """ + get difference version of package in specfic repo + """ + def __init__(self, log_file_name=''): + """ + """ + if local_conf is None: + logging.error("local_conf is None") + return + self.sender = dict(local_conf.conf.items("sender")) + self.cc = dict(local_conf.conf.items("cc")) + self.old_repo = dict(local_conf.conf.items("old_repo")) + self.new_repo = dict(local_conf.conf.items("new_repo")) + self.yum = dict(local_conf.conf.items("yum")) + self.local_data = dict(local_conf.conf.items("local_data")) + self.repo_old = self.old_repo.get('repos').split(',') + self.repo_new = self.new_repo.get('repos').split(',') + + # record receiver's email + self.to_email = [] + + self.pkgs_result = [] + self.pkgs = [] + self.pkgs_version_old = dict() + self.pkgs_repo_old = dict() + self.pkgs_version_new = dict() + self.pkgs_repo_new = dict() + + def process(self): + """ + Preparation and processing of the main entrance. + """ + # 是否读取本地的repolist数据 + load_local_data_flag = self.local_data.get('load_local_data', 'No') + if self._str_equal(load_local_data_flag, "Yes"): + old_repolist_file = self.local_data.get('old_repolist_file') + new_repolist_file = self.local_data.get('new_repolist_file') + else: + old_repolist_file = '{}.csv'.format(self.repo_old[0]) + new_repolist_file = '{}.csv'.format(self.repo_new[0]) + + with open(old_repolist_file, "w+") as file1: + file1.truncate() + with open(new_repolist_file, "w+") as file2: + file2.truncate() + + if not self._set_repo(): + return False + + self.pkgs, self.pkgs_version_old, self.pkgs_repo_old = self._get_srpm_info(self.repo_old, old_repolist_file) + pkgs_new, self.pkgs_version_new, self.pkgs_repo_new = self._get_srpm_info(self.repo_new, new_repolist_file) + + self._get_all_result() + self.notify_all_respon_person() + return True + + def _set_repo(self): + """ + Back up the repo directory and configure a new repo + + returns: + True: Successful execution + False: Execution failed + """ + repo_path = self.yum.get('repo_path') + from_repo = self.yum.get('repo_file') + + if not mv_files(repo_path, self.yum.get('back_repo_path')): + logging.error("back repo file to temprepo failed!") + return False + + if not copy_file(from_repo, repo_path): + logging.error(f"set the repo cofigfile: {from_repo} failed!") + return False + + return True + + def _get_srpm_info(self, repolist, repolist_file): + """ + get srpm info from given repo + + args: + repolist: defined repo name list + repolist_file: defined file name for storing repolist results of repolist + return: + pkgs: package name list + pkgs_version: dict of which the key value pair is the package name and version number + pkgs_repo: dict of which the key value pair is the package name and repo name + """ + # 基于yum list --repo的结果有三列 + srpm_info_num = 3 + # 存放srpm解析得到的包名，例如：gcc-9.3.1-20210628.21.oe1.src.rpm ——> gcc + pkgs = set() + # 存放srpm解析得到的（包名，版本号）键值对字典，例如：gcc-9.3.1-20210628.21.oe1.src.rpm ——> ('gcc', '9.3.1-20210628.21') + pkgs_version = dict() + # 存放srpm解析得到的（包名，repo名）键值对字典，例如：gcc-9.3.1-20210628.21.oe1.src.rpm SP2 ——> ('gcc', 'SP2') + pkgs_repo = dict() + num = 0 + with codecs.open(repolist_file, 'w', 'utf-8') as result_file: + writer = csv.writer(result_file) + writer.writerow(["Package", "Version_release", "Reponame"]) + for repo in repolist: + logging.info(f"yum list repo: {repo} >> {repolist_file}") + yum_list_cmd = f"yum list --repo {repo}" + cmd_out = ExecuteCmd().cmd_output(yum_list_cmd.split()) + + if cmd_out is None: + logging.error(f"yum list --repo {repo} failed") + return [] + data = cmd_out.split("\n") + + for line in data: + if '.src' in line: + num = num+1 + # 清理yum list --repo返回信息中的不必要信息：.src/.oe1 + line = line.replace('.src', '') + line = line.replace('.oe1', '') + srpm_list = line.split() + if len(srpm_list) == srpm_info_num: + pkg_name = srpm_list[0] + pkgs.add(pkg_name) + pkgs_version[pkg_name] = srpm_list[1] + pkgs_repo[pkg_name] = srpm_list[2] + # example: CUnit,2.1.3-22,SP1_realtime + writer.writerow(srpm_list) + + logging.info(f"{num}") + return list(pkgs), pkgs_version, pkgs_repo + + def _get_all_result(self): + """ + get the difference of the corresponding version number of the package + """ + result_file_name = 'result_{}_{}.csv'.format(self.repo_old[0], self.repo_new[0]) + + with codecs.open(result_file_name, 'w', 'utf-8') as result_file: + writer = csv.writer(result_file) + writer.writerow(["Package", self.repo_old[0], self.repo_new[0], "Project", "Developer"]) + + for pkg in self.pkgs: + pkg_version_old = self.pkgs_version_old.get(pkg, 'NA') + pkg_repo_old = self.pkgs_repo_old.get(pkg, 'NA') + pkg_version_new = self.pkgs_version_new.get(pkg, 'NA') + + if self._version_diff(pkg_version_old, pkg_version_new): + result_data = [] + logging.info(f"Find {pkg}: {pkg_version_old} > {pkg_version_new}") + + result_data.append(pkg) + result_data.append(pkg_version_old) + result_data.append(pkg_version_new) + result_data.append(pkg_repo_old) + + # 获取软件包的sig组信息、最近的贡献者邮箱信息 + sig = MaintainerAndContributor(self.old_repo.get('branch')) + real_package = sig.get_package_repo_name(pkg) + sig_name = sig.get_manager_sig(real_package) + developer_emails = sig.get_latest_contributors(real_package) + sig_maintainers = sig.get_sig_maintainers(sig_name) + + result_data.append(developer_emails[0]) + self.to_email.append(developer_emails[0]) + + epol_proj = 'EPOL' + if epol_proj in pkg_repo_old: + project = self.old_repo.get('epol_project') + else: + project = self.old_repo.get('main_project') + + single_pkg_res = { + "pkg_name": pkg, + "version_old": pkg_version_old, + "version_new": pkg_version_new, + "project": project, + "sig": sig_name, + "maintainers": sig_maintainers[0], + "developer_email": developer_emails[0] + } + logging.info(f"{single_pkg_res}") + + writer.writerow(result_data) + self.pkgs_result.append(single_pkg_res) + + logging.info("process compare pkglist end!") + return + + def _version_diff(self, x_version, y_version): + """ + Compare the size of the two version numbers + """ + version_array_x=re.split(r"\.|\:|\-", x_version) + version_array_y=re.split(r"\.|\:|\-", y_version) + + len_x_version_array = len(version_array_x) + len_y_version_array = len(version_array_y) + + len_min = min(len_x_version_array, len_y_version_array) + + for i in range(len_min): + x = version_array_x[i] + y = version_array_y[i] + if x.isdigit() and y.isdigit(): + x = int(x) + y = int(y) + if y < x: + return True + elif y > x: + return False + if i == len_y_version_array - 1: + return False + + def _str_equal(self, str_left, str_right): + """ + Compare whether two strings of the same length are equal + + args: + str_left/str_right: the input string + """ + if isinstance(str_left, str): + if isinstance(str_right, str): + return len(str_left) == len(str_right) and str_left.upper() == str_right.upper() + return False + + def notify_all_respon_person(self): + """ + notice all packages to person + """ + message = self._edit_email_content() + if self._send_email(message): + logging.info("send email succeed !") + return True + else: + logging.error(f"send email failed!") + return False + + def _edit_email_content(self): + """ + edit email content + """ + line = "" + for single_pkg_res in self.pkgs_result: + pkg_url = "https://build.openeuler.org/package/show/%s/%s" % (single_pkg_res.get("project"), \ + single_pkg_res.get("pkg_name")) + line = line + """ + %s%s%s%s%s%s%s + """ % ( + single_pkg_res.get("pkg_name"), single_pkg_res.get("version_old"), + single_pkg_res.get("version_new"), pkg_url, + single_pkg_res.get("project"), single_pkg_res.get("sig"), + single_pkg_res.get("maintainers"), single_pkg_res.get("developer_email") + ) + message = """ +

Hello:

+ + + + %s +
Package%s%sOBS ProjectSigMaintainer(1st)Latest Contributor's Email
+

Please solve it as soon as possible.

+

Thanks ~^v^~ !!!

+ """ % (self.repo_old[0], self.repo_new[0], line) + return message + + def _send_email(self, message): + """ + send a email + + args: + message: Message body + returns: + True: send email normally + False: Abnormal email sending feedback + """ + sender_email = self.sender.get('email') + sender_email_pass = self.sender.get('pass') + msg = MIMEText(message, 'html') + msg['Subject'] = Header("[OBS Package Version Difference Notice]", "utf-8") + msg["From"] = Header(self.sender.get('email')) + + cc_emails = self.cc.get('email').split(',') + msg["To"] = Header(cc_emails[0]) + msg["Cc"] = Header(cc_emails[0]) + + to_email = self._check_email() + + try: + server = smtplib.SMTP_SSL(self.sender.get('smtp_server')) + server.login(sender_email, sender_email_pass) + server.sendmail(sender_email, to_email + cc_emails, msg.as_string()) + server.quit() + return True + except smtplib.SMTPException as err: + logging.error(f"Abnormal email sending feedback: {err}") + return False + + def _check_email(self): + """ + Verify and keep the legal email address + + returns: + valid_email: legal email address + """ + valid_email = set() + for origin_email in self.to_email: + try: + regular = re.compile(r'[0-9a-zA-Z\.]+@[0-9a-zA-Z\.]+[com, org]') + email = re.findall(regular, origin_email)[0] + if email: + valid_email.add(email) + except IndexError as e: + logging.error(f"analyse developer for {email} failed") + return list(valid_email) + + +if __name__ == '__main__': + pvd = PkgVersionDiff() + pvd.process() \ No newline at end of file -- Gitee From 4efdb3f580fc5154539531009de94b7ce1be6bac Mon Sep 17 00:00:00 2001 From: small_leek Date: Mon, 26 Jul 2021 18:54:27 +0800 Subject: [PATCH 2/2] init common class py add sig_maintainer.py add sig_maintainer.py --- release-assistant/javcra/api/executecmd.py | 60 +++ release-assistant/javcra/api/loadconf.py | 82 ++++ release-assistant/javcra/api/logger.py | 61 +++ .../javcra/api/sig_maintainer.py | 256 +++++++++++++ release-assistant/javcra/api/versiondiff.py | 354 ------------------ 5 files changed, 459 insertions(+), 354 deletions(-) create mode 100644 release-assistant/javcra/api/executecmd.py create mode 100644 release-assistant/javcra/api/loadconf.py create mode 100644 release-assistant/javcra/api/logger.py create mode 100644 release-assistant/javcra/api/sig_maintainer.py delete mode 100644 release-assistant/javcra/api/versiondiff.py diff --git a/release-assistant/javcra/api/executecmd.py b/release-assistant/javcra/api/executecmd.py new file mode 100644 index 0000000..66cb736 --- /dev/null +++ b/release-assistant/javcra/api/executecmd.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-07-3 +# ******************************************************************************/ + +import os +import subprocess +from logger import log_check +import logging + +class ExecuteCmd(object): + """ + """ + def __init__(self): + """ + """ + + def cmd_status(self, command): + """ + Execute command and return to status + + args: + command: command to be executed + returns: + subprocess.run(xxx).returncode + """ + ret = subprocess.run(command, shell=False, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf-8", timeout=10) + if ret.returncode: + logging.info(f"error:{ret}") + + return ret.returncode + + def cmd_output(self, command): + """ + Execute the command and return the output + + args: + command: command to be executed + returns: + subprocess.check_output(xxx) + """ + try: + subp = subprocess.check_output(command, shell=False, stderr=subprocess.STDOUT, encoding="utf-8") + return subp + except subprocess.CalledProcessError as err: + logging.error(f"{command}:{err}") + return None + \ No newline at end of file diff --git a/release-assistant/javcra/api/loadconf.py b/release-assistant/javcra/api/loadconf.py new file mode 100644 index 0000000..cf702ad --- /dev/null +++ b/release-assistant/javcra/api/loadconf.py @@ -0,0 +1,82 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-07-04 +# ******************************************************************************/ + +import configparser +import os +from logger import log_check +import logging + +class LoadConfig(object): + """ + """ + def __init__(self): + """ + """ + src_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + properties_path = src_dir + "/config/properties.conf" + self.conf = self.read_conf(properties_path) + + def read_conf(self, properties_path): + """ + args: + properties_path: path of configuration file + returns: + config: the object of configparser.ConfigParser() + """ + if not os.path.isfile(properties_path): + logging.error(f"Invalid file path: {properties_path}") + return None + try: + config = configparser.ConfigParser() + config.read(properties_path, encoding="utf8") + return config + except configparser.NoSectionError as err: + logging.error(f"{properties_path} is invalid!") + return None + + def get_section_key_value(self, section, key): + """ + Gets the value of the key in the specified section + + args: + section: section + key: key + + returns: + value: the value of the key in the specified section + """ + try: + value = self.conf.get(section, key) + return value + except AttributeError as err: + logging.error(f"reason: {err}") + return "NA" + except KeyError as err: + logging.error(f"reason: {err}") + return "NA" + except configparser.NoOptionError as err: + logging.error(f"reason: {err}") + return "NA" + except configparser.NoSectionError as err: + logging.error(f"reason: {err}") + return "NA" + +local_conf = LoadConfig() + +if __name__ == "__main__": + sig_query_conf = dict(local_conf.conf.items("sig_query_conf")) + sigs_yaml_url = sig_query_conf.get('sigs_yaml_url') + logging.info(f"{sigs_yaml_url}") \ No newline at end of file diff --git a/release-assistant/javcra/api/logger.py b/release-assistant/javcra/api/logger.py new file mode 100644 index 0000000..d205915 --- /dev/null +++ b/release-assistant/javcra/api/logger.py @@ -0,0 +1,61 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2020-08-10 +# ******************************************************************************/ +""" +log module：logger.py +""" +import configparser +import datetime +import logging +import os + +class Logger(object): + """ + """ + def __init__(self, name=__name__): + """ + """ + # name 即 module + self.log_object = logging.getLogger(name) + # 创建存放日志的目录 + if not os.path.exists('./TestLog'): + os.mkdir('./TestLog') + + def get_logger(self, log_file_name=''): + """ + args: + log_file_name: Add the log file name of the file output stream. It is empty by default + returns: + log_object + """ + nowTime = datetime.datetime.now().strftime('%Y-%m-%d') + # 若外部接口调用时指定了日志文件，则以log_file_name为准 + if not log_file_name: + file_name = f'./TestLog/{log_file_name}_{nowTime}.log' + + file_name = f'./TestLog/{nowTime}.log' + file_handler = logging.FileHandler(filename=file_name, encoding='utf-8', mode='a') + + self.log_object.handlers = [] + self.log_object.addHandler(file_handler) + logging.basicConfig(level=logging.DEBUG, + format='[%(asctime)s][%(levelname)s][%(module)s:%(lineno)d|%(funcName)s] %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + return self.log_object + +log_check = Logger(__name__).get_logger() + diff --git a/release-assistant/javcra/api/sig_maintainer.py b/release-assistant/javcra/api/sig_maintainer.py new file mode 100644 index 0000000..10829e0 --- /dev/null +++ b/release-assistant/javcra/api/sig_maintainer.py @@ -0,0 +1,256 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-06-20 +# ******************************************************************************/ +""" +This is a simple script to query that contact person for specific package +""" +import argparse +import os +import re +import shutil +import logging +from executecmd import ExecuteCmd +from loadconf import local_conf +from logger import log_check +from types import TracebackType +import urllib.request +import yaml + +class MaintainerAndContributor(object): + """ + get sig maintainer and contributor of package + """ + def __init__(self, bra, log_file_name=''): + """ + args: + bra: specfilc branch for spec file + log_obj: input log object + """ + self.branch = bra + if local_conf is None: + logging.error(f"local_conf is None") + return + self.sig_query_conf = dict(local_conf.conf.items("sig_query_conf")) + + + def _get_gitee(self, url): + """ + get the content of the file pointed to by the url + args: + url: url of specific file on gitee + + returns: content of HTTPResposne object + """ + headers = eval(self.sig_query_conf.get('headers')) + try: + req = urllib.request.Request(url = url, headers = headers) + u = urllib.request.urlopen(req) + html = u.read().decode("utf-8") + return html + except urllib.error.HTTPError as err: + logging.error(f"{url} reason: {err.reason}") + except urllib.error.URLError as err: + logging.error(f"{url} reason: {err.reason}") + except ValueError as err: + logging.error(f"{url} reason: {err}") + return None + return None + + def get_sigs_yaml(self): + """ + get the content of the sigs.yaml + + returns: + sigs: content of sigs.yaml + """ + try: + sigs_yaml_url = self.sig_query_conf.get('sigs_yaml_url') + headers = eval(self.sig_query_conf.get('headers')) + req = urllib.request.Request(url = sigs_yaml_url, headers = headers) + u = urllib.request.urlopen(req) + html = u.read().decode("utf-8") + except urllib.error.HTTPError as err: + logging.error(f"{sigs_yaml_url} reason: {err.reason}") + return None + except ValueError as err: + logging.error(f"{sigs_yaml_url} reason: {err}") + return None + sigs = yaml.load(html, Loader=yaml.Loader) + return sigs + + + def get_package_repo_name(self, srpm_name): + """ + get the real package repo name on gitee + + args: + srpm_name: the name of the srpm + returns: + package: the name of repository on https://gitee.com/src-openeuler/ + """ + package = srpm_name + with open(self.sig_query_conf.get('local_specfile_exception'), encoding='utf-8') as f: + exps = yaml.load(f, Loader=yaml.Loader) + for key in exps: + if srpm_name == exps[key].get("srpm"): + package = key + logging.info(f"real package:{package}") + break + #logging.info(f"real package:{package}") + return package + + + def get_manager_sig(self, package): + """ + get the sig name of specific package + + args: + pacakge + + returns: + sig_name: SIG name to which package belongs + """ + sig_name = "NA" + sig_load = self.get_sigs_yaml() + for sig in sig_load.get("sigs"): + for repo in sig.get("repositories"): + if repo == "src-openeuler/" + package: + return sig.get("name", 'NA') + return sig_name + + def get_owner_sigs(self, gitee_id): + """ + get the sigs maintained by gitee_id + + args: + gitee_id: Registered gitee ID + returns: + own_sigs: Maintained sig list + """ + own_sigs = [] + sig_load = self.get_sigs_yaml() + sigs = sig_load.get("sigs") + + for sig_names in sigs: + sig_name = sig_names.get("name") + sig_owners = self.get_sig_maintainers(sig_name) + if gitee_id in sig_owners: + own_sigs.append(sig_name) + return own_sigs + + + def get_sig_maintainers(self, sig_name): + """ + get maintainers of specific sig + + args: + sig_name: name of sig + returns: + maintainers: maintainers of sig + """ + maintainers = [] + if sig_name is None: + maintainers = ['NA'] + return maintainers + url = self.sig_query_conf.get('sig_owner_url_template').format(signame = sig_name) + r = self._get_gitee(url) + if r is None: + return maintainers + else: + owners = yaml.load(r, Loader = yaml.Loader) + return owners.get("maintainers", maintainers) + + def get_latest_contributors(self, package, max_eamil_num = 3): + """ + get latest contributors's emails + + args: + package: package name/spec filename + max_eamil_num: limit the maximum number of eamils + returns: + emails: eamils of latest contributors + """ + emails = [] + specurl = self.sig_query_conf.get('spec_file_url_template').format( + package = package, + branch = self.branch, + specfile = package + ".spec") + + spec = self._get_gitee(specurl) + if spec is None: + logging.debug(f"urllib.request failed and try to curl: {package}.spec") + spec_file_path = self.curl_pkg_spec(package) + if spec_file_path != '': + spec = spec_file_path.read().decode("utf-8") + else: + logging.error("get spec of %s failed!", package) + emails = ['NA'] + return emails + + return self.get_emails_of_contributors(spec, max_eamil_num, emails) + + + def get_emails_of_contributors(self, spec, max_eamil_num, emails): + """ + analyse the email of contributor in changelog + args: + spec: content of spec file + max_eamil_num: limit the maximum number of eamils + returns: + emails: eamils of latest max_eamil_num contributors + """ + num = 0 + in_changelog = False + for line in spec.splitlines(): + if line.startswith("%changelog"): + in_changelog = True + if in_changelog and line.startswith("*") and num < max_eamil_num: + try: + regular = re.compile(r'[0-9a-zA-Z\.]+@[0-9a-zA-Z\.]+[com, org]') + email = re.findall(regular, line)[0] + emails.append(email) + num = num + 1 + except IndexError as e: + logging.error(f"analyse developer for {line} failed") + emails.append(line) + if not emails: + return ['NA'] + return emails + + + def curl_pkg_spec(self, package): + """ + download specfile using curl command + args: + package: package name in gitee + return: + pkg_path: file path after successful download + """ + exec = ExecuteCmd() + mktmp_cmd = "mktemp -d" + _tmpdir = exec.cmd_output(mktmp_cmd.split()).strip('\n') + if _tmpdir is None: + logging.error(f"mktemp failed") + curl_cmd = f"curl -o {_tmpdir}/{package}.spec \ + https://gitee.com/src-openeuler/{package}/raw/{self.branch}/{package}.spec &>/dev/null" + cmd_out = exec.cmd_status(curl_cmd.split()) + if cmd_out: + shutil.rmtree(_tmpdir) + return '' + else: + return f"{_tmpdir}/{package}.spec" + + + diff --git a/release-assistant/javcra/api/versiondiff.py b/release-assistant/javcra/api/versiondiff.py deleted file mode 100644 index 95e7eea..0000000 --- a/release-assistant/javcra/api/versiondiff.py +++ /dev/null @@ -1,354 +0,0 @@ -#! /usr/bin/env python -# coding=utf-8 -# ****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Author: senlin -# Create: 2021-05-27 -# ******************************************************************************/ - -import argparse -import csv -import codecs - -from executecmd import ExecuteCmd -from file_manage import mv_files -from file_manage import copy_file -from loadconf import local_conf -import logging -import os -import re -import shutil -import smtplib -from sig_maintainer import MaintainerAndContributor -from itertools import islice -from email.mime.text import MIMEText -from email.header import Header -from pathlib import Path -from loadconf import LoadConfig - -class PkgVersionDiff(object): - """ - get difference version of package in specfic repo - """ - def __init__(self, log_file_name=''): - """ - """ - if local_conf is None: - logging.error("local_conf is None") - return - self.sender = dict(local_conf.conf.items("sender")) - self.cc = dict(local_conf.conf.items("cc")) - self.old_repo = dict(local_conf.conf.items("old_repo")) - self.new_repo = dict(local_conf.conf.items("new_repo")) - self.yum = dict(local_conf.conf.items("yum")) - self.local_data = dict(local_conf.conf.items("local_data")) - self.repo_old = self.old_repo.get('repos').split(',') - self.repo_new = self.new_repo.get('repos').split(',') - - # record receiver's email - self.to_email = [] - - self.pkgs_result = [] - self.pkgs = [] - self.pkgs_version_old = dict() - self.pkgs_repo_old = dict() - self.pkgs_version_new = dict() - self.pkgs_repo_new = dict() - - def process(self): - """ - Preparation and processing of the main entrance. - """ - # 是否读取本地的repolist数据 - load_local_data_flag = self.local_data.get('load_local_data', 'No') - if self._str_equal(load_local_data_flag, "Yes"): - old_repolist_file = self.local_data.get('old_repolist_file') - new_repolist_file = self.local_data.get('new_repolist_file') - else: - old_repolist_file = '{}.csv'.format(self.repo_old[0]) - new_repolist_file = '{}.csv'.format(self.repo_new[0]) - - with open(old_repolist_file, "w+") as file1: - file1.truncate() - with open(new_repolist_file, "w+") as file2: - file2.truncate() - - if not self._set_repo(): - return False - - self.pkgs, self.pkgs_version_old, self.pkgs_repo_old = self._get_srpm_info(self.repo_old, old_repolist_file) - pkgs_new, self.pkgs_version_new, self.pkgs_repo_new = self._get_srpm_info(self.repo_new, new_repolist_file) - - self._get_all_result() - self.notify_all_respon_person() - return True - - def _set_repo(self): - """ - Back up the repo directory and configure a new repo - - returns: - True: Successful execution - False: Execution failed - """ - repo_path = self.yum.get('repo_path') - from_repo = self.yum.get('repo_file') - - if not mv_files(repo_path, self.yum.get('back_repo_path')): - logging.error("back repo file to temprepo failed!") - return False - - if not copy_file(from_repo, repo_path): - logging.error(f"set the repo cofigfile: {from_repo} failed!") - return False - - return True - - def _get_srpm_info(self, repolist, repolist_file): - """ - get srpm info from given repo - - args: - repolist: defined repo name list - repolist_file: defined file name for storing repolist results of repolist - return: - pkgs: package name list - pkgs_version: dict of which the key value pair is the package name and version number - pkgs_repo: dict of which the key value pair is the package name and repo name - """ - # 基于yum list --repo的结果有三列 - srpm_info_num = 3 - # 存放srpm解析得到的包名，例如：gcc-9.3.1-20210628.21.oe1.src.rpm ——> gcc - pkgs = set() - # 存放srpm解析得到的（包名，版本号）键值对字典，例如：gcc-9.3.1-20210628.21.oe1.src.rpm ——> ('gcc', '9.3.1-20210628.21') - pkgs_version = dict() - # 存放srpm解析得到的（包名，repo名）键值对字典，例如：gcc-9.3.1-20210628.21.oe1.src.rpm SP2 ——> ('gcc', 'SP2') - pkgs_repo = dict() - num = 0 - with codecs.open(repolist_file, 'w', 'utf-8') as result_file: - writer = csv.writer(result_file) - writer.writerow(["Package", "Version_release", "Reponame"]) - for repo in repolist: - logging.info(f"yum list repo: {repo} >> {repolist_file}") - yum_list_cmd = f"yum list --repo {repo}" - cmd_out = ExecuteCmd().cmd_output(yum_list_cmd.split()) - - if cmd_out is None: - logging.error(f"yum list --repo {repo} failed") - return [] - data = cmd_out.split("\n") - - for line in data: - if '.src' in line: - num = num+1 - # 清理yum list --repo返回信息中的不必要信息：.src/.oe1 - line = line.replace('.src', '') - line = line.replace('.oe1', '') - srpm_list = line.split() - if len(srpm_list) == srpm_info_num: - pkg_name = srpm_list[0] - pkgs.add(pkg_name) - pkgs_version[pkg_name] = srpm_list[1] - pkgs_repo[pkg_name] = srpm_list[2] - # example: CUnit,2.1.3-22,SP1_realtime - writer.writerow(srpm_list) - - logging.info(f"{num}") - return list(pkgs), pkgs_version, pkgs_repo - - def _get_all_result(self): - """ - get the difference of the corresponding version number of the package - """ - result_file_name = 'result_{}_{}.csv'.format(self.repo_old[0], self.repo_new[0]) - - with codecs.open(result_file_name, 'w', 'utf-8') as result_file: - writer = csv.writer(result_file) - writer.writerow(["Package", self.repo_old[0], self.repo_new[0], "Project", "Developer"]) - - for pkg in self.pkgs: - pkg_version_old = self.pkgs_version_old.get(pkg, 'NA') - pkg_repo_old = self.pkgs_repo_old.get(pkg, 'NA') - pkg_version_new = self.pkgs_version_new.get(pkg, 'NA') - - if self._version_diff(pkg_version_old, pkg_version_new): - result_data = [] - logging.info(f"Find {pkg}: {pkg_version_old} > {pkg_version_new}") - - result_data.append(pkg) - result_data.append(pkg_version_old) - result_data.append(pkg_version_new) - result_data.append(pkg_repo_old) - - # 获取软件包的sig组信息、最近的贡献者邮箱信息 - sig = MaintainerAndContributor(self.old_repo.get('branch')) - real_package = sig.get_package_repo_name(pkg) - sig_name = sig.get_manager_sig(real_package) - developer_emails = sig.get_latest_contributors(real_package) - sig_maintainers = sig.get_sig_maintainers(sig_name) - - result_data.append(developer_emails[0]) - self.to_email.append(developer_emails[0]) - - epol_proj = 'EPOL' - if epol_proj in pkg_repo_old: - project = self.old_repo.get('epol_project') - else: - project = self.old_repo.get('main_project') - - single_pkg_res = { - "pkg_name": pkg, - "version_old": pkg_version_old, - "version_new": pkg_version_new, - "project": project, - "sig": sig_name, - "maintainers": sig_maintainers[0], - "developer_email": developer_emails[0] - } - logging.info(f"{single_pkg_res}") - - writer.writerow(result_data) - self.pkgs_result.append(single_pkg_res) - - logging.info("process compare pkglist end!") - return - - def _version_diff(self, x_version, y_version): - """ - Compare the size of the two version numbers - """ - version_array_x=re.split(r"\.|\:|\-", x_version) - version_array_y=re.split(r"\.|\:|\-", y_version) - - len_x_version_array = len(version_array_x) - len_y_version_array = len(version_array_y) - - len_min = min(len_x_version_array, len_y_version_array) - - for i in range(len_min): - x = version_array_x[i] - y = version_array_y[i] - if x.isdigit() and y.isdigit(): - x = int(x) - y = int(y) - if y < x: - return True - elif y > x: - return False - if i == len_y_version_array - 1: - return False - - def _str_equal(self, str_left, str_right): - """ - Compare whether two strings of the same length are equal - - args: - str_left/str_right: the input string - """ - if isinstance(str_left, str): - if isinstance(str_right, str): - return len(str_left) == len(str_right) and str_left.upper() == str_right.upper() - return False - - def notify_all_respon_person(self): - """ - notice all packages to person - """ - message = self._edit_email_content() - if self._send_email(message): - logging.info("send email succeed !") - return True - else: - logging.error(f"send email failed!") - return False - - def _edit_email_content(self): - """ - edit email content - """ - line = "" - for single_pkg_res in self.pkgs_result: - pkg_url = "https://build.openeuler.org/package/show/%s/%s" % (single_pkg_res.get("project"), \ - single_pkg_res.get("pkg_name")) - line = line + """ - %s%s%s%s%s%s%s - """ % ( - single_pkg_res.get("pkg_name"), single_pkg_res.get("version_old"), - single_pkg_res.get("version_new"), pkg_url, - single_pkg_res.get("project"), single_pkg_res.get("sig"), - single_pkg_res.get("maintainers"), single_pkg_res.get("developer_email") - ) - message = """ -

Hello:

- - - - %s -
Package%s%sOBS ProjectSigMaintainer(1st)Latest Contributor's Email
-

Please solve it as soon as possible.

-

Thanks ~^v^~ !!!

- """ % (self.repo_old[0], self.repo_new[0], line) - return message - - def _send_email(self, message): - """ - send a email - - args: - message: Message body - returns: - True: send email normally - False: Abnormal email sending feedback - """ - sender_email = self.sender.get('email') - sender_email_pass = self.sender.get('pass') - msg = MIMEText(message, 'html') - msg['Subject'] = Header("[OBS Package Version Difference Notice]", "utf-8") - msg["From"] = Header(self.sender.get('email')) - - cc_emails = self.cc.get('email').split(',') - msg["To"] = Header(cc_emails[0]) - msg["Cc"] = Header(cc_emails[0]) - - to_email = self._check_email() - - try: - server = smtplib.SMTP_SSL(self.sender.get('smtp_server')) - server.login(sender_email, sender_email_pass) - server.sendmail(sender_email, to_email + cc_emails, msg.as_string()) - server.quit() - return True - except smtplib.SMTPException as err: - logging.error(f"Abnormal email sending feedback: {err}") - return False - - def _check_email(self): - """ - Verify and keep the legal email address - - returns: - valid_email: legal email address - """ - valid_email = set() - for origin_email in self.to_email: - try: - regular = re.compile(r'[0-9a-zA-Z\.]+@[0-9a-zA-Z\.]+[com, org]') - email = re.findall(regular, origin_email)[0] - if email: - valid_email.add(email) - except IndexError as e: - logging.error(f"analyse developer for {email} failed") - return list(valid_email) - - -if __name__ == '__main__': - pvd = PkgVersionDiff() - pvd.process() \ No newline at end of file -- Gitee