Open Build Service

We truncated the diff of some files because they were too big. If you want to see the full diff for every file, click here.

Changes of Revision 2

build_result.py Added

@@ -0,0 +1,208 @@
+#/bin/env python3
+# -*- encoding=utf8 -*-
+#******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ******************************************************************************
+
+
+import os
+import csv
+import logging
+import argparse
+import subprocess
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor
+
+
+def run(cmd, timeout=600, print_out=False):
+    """run shell cmd"""
+    ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8",
+                         timeout=timeout)
+    logging.info("cmd: {}".format(cmd))
+    if ret.stdout and print_out:
+        logging.info("ret.stdout: {}".format(ret.stdout))
+    if ret.stderr:
+        logging.warning("ret.stderr: {}".format(ret.stderr))
+    return ret.returncode, ret.stdout, ret.stderr
+
+
+def write_to_csv(csv_path, build_result_data):
+    """write build result data to csv"""
+    if not build_result_data:
+        logging.warning(f"{csv_path} has nothing to write")
+        return
+
+    with open(csv_path, mode='w', newline="\n") as csv_file:
+        fieldnames = list(build_result_data[0].keys())
+        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+        writer.writeheader()
+        for build_result in build_result_data:
+            writer.writerow(build_result)
+
+
+class ProjectBuildResult:
+    """project build result"""
+
+    def __init__(self, project, **kwargs):
+        """param init"""
+        self.kwargs = kwargs
+        self.project = project
+        self.result_xml = self.kwargs.get("result_xml", "")
+        self.published_arch_list = []   # published repository list   eg: [aarch64, x86_64]
+        self.build_result_list = []
+
+    def parse_build_result(self):
+        """parse build result, update project status and packages status"""
+        try:
+            if self.result_xml and os.path.exists(self.result_xml):
+                xml_tree = ET.parse(self.result_xml)
+                root = xml_tree.getroot()
+            else:
+                cmd = f"osc results {self.project} --xml"
+                _, out, _ = run(cmd)
+                root = ET.fromstring(out)
+        except Exception as e:
+            logging.error(f"{self.project} parse Exception: {e}")
+            return False
+
+        for repo_result in root:
+            arch = repo_result.attrib["arch"]
+            build_status = repo_result.attrib["code"]     # "published"
+
+            # if published, append self.published_repo_list
+            if build_status == "published":
+                self.published_arch_list.append(arch)
+
+            # update package build result
+            for pkg_result in repo_result:
+                pkg_result_dict = {}
+                package = pkg_result.attrib["package"]
+                build_result = pkg_result.attrib["code"]
+                details = ""
+                if build_result == "unresolvable":
+                    for ele in pkg_result:
+                        details = ele.text
+
+                pkg_result_dict["project"] = self.project
+                pkg_result_dict["package"] = package
+                pkg_result_dict["arch"] = arch
+                pkg_result_dict["build_result"] = build_result
+                pkg_result_dict["details"] = details
+                self.build_result_list.append(pkg_result_dict)
+        return True
+
+    def check_project_is_published(self):
+        """check project repository is published or not"""
+        try:
+            if not self.parse_build_result():
+                return False
+        except Exception as e:
+            logging.error(f"parse {self.project} build result Exception: {e}")
+            return False
+        if self.published_repo_list:
+            return True
+        return False
+
+    def get_failed_unresolvable_list(self):
+        """get failed and unresolvable package list"""
+        failed_unresolvable_list = []
+        for pkg_build_result in self.build_result_list:
+            if (pkg_build_result["build_result"] == "failed") or (pkg_build_result["build_result"] == "unresolvable"):
+                failed_unresolvable_list.append(pkg_build_result)
+        return failed_unresolvable_list
+
+    def check_use_for_build(self):
+        """check project use for build flag everyday"""
+        # 1. parse build result
+        try:
+            self.parse_build_result()
+        except Exception as e:
+            logging.error(f"parse {self.project} build result Exception: {e}")
+            return False
+
+        with ThreadPoolExecutor(50) as executor:
+            # 1. check use for build flag for succeeded package: open
+            for repo, succeeded_pkg_list in self.succeeded_pkg_dict.items():
+                for pkg in succeeded_pkg_list:
+                    executor.submit(lambda p: self.pkg_check_use_for_build(*p), (repo, pkg, "open"))
+
+            # 1. check use for build flag for failed package: close
+            for repo, failed_pkg_list in self.failed_pkg_dict.items():
+                for pkg in failed_pkg_list:
+                    executor.submit(lambda p: self.pkg_check_use_for_build(*p), (repo, pkg, "close"))
+
+        if self.check_flag_not_correct_list:
+            logging.error(f"{self.project} use for build flag not correct list: {self.check_flag_not_correct_list}")
+            return False
+        else:
+            logging.info(f"{self.project} use for build flag check no problem!")
+        logging.info(f"{self.project} use for build flag check finish!")
+        return True
+
+
+if __name__ == "__main__":
+    par = argparse.ArgumentParser()
+    par.add_argument("-p", "--project", default="", nargs="+", help="obs project", required=False)
+    par.add_argument("-pf", "--project_list_file", default="", help="obs project list file", required=False)
+    par.add_argument("-r", "--result_xml", default="", help="obs project build result xml", required=False)
+    par.add_argument("-w", "--write_to_csv", default=1, help="obs project build result xml", required=False)
+    par.add_argument("-c", "--csv_path", default="", help="obs project build result write to csv path", required=False)
+    par.add_argument("-lf", "--log_to_file", default=0, help="print log to file", required=False)
+    args = par.parse_args()
+
+    kw = {
+        "result_xml": args.result_xml,
+    }
+
+    LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s"
+    DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+    if args.log_to_file:
+        log_file = os.path.realpath(__file__).replace(".py", ".log")
+        logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT, filename=log_file,
+                            filemode="a")
+    else:
+        logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT)
+
+    project_list = args.project
+    if os.path.exists(args.project_list_file):
+        with open(args.project_list_file, "r") as f:
+            project_list = f.read().split()
+
+    exit_code = 0
+    # project_list = ["openEuler:Mainline"]
+    # kw["result_xml"] = "openeuler_mainline_result_20220706.xml"
+    project_list = ["openEuler:Epol"]
+    kw["result_xml"] = "openeuler_epol_result_20220706.xml"
+    all_failed_unresolvable_pkgs = []
+    for obs_project in project_list:
+        project_build = ProjectBuildResult(obs_project, **kw)
+        if not project_build.parse_build_result():
+            logging.error(f"project {obs_project} build result parse failed")
+            exit_code = 1
+        else:
+            failed_unresolvable_pkgs = project_build.get_failed_unresolvable_list()
+            if failed_unresolvable_pkgs:
+                logging.warning(f"{obs_project} failed unresolvable packages num: {len(failed_unresolvable_pkgs)}")
+                logging.warning(f"{obs_project} failed unresolvable packages list: {failed_unresolvable_pkgs}")
+                all_failed_unresolvable_pkgs.extend(failed_unresolvable_pkgs)
+
+    if all_failed_unresolvable_pkgs:
+        logging.warning(f"final failed unresolvable packages num: {len(all_failed_unresolvable_pkgs)}")
+        logging.warning(f"final failed unresolvable packages list: {all_failed_unresolvable_pkgs}")