[enhancement] replace NTHREADS threading control with MAKEFLAGS, remove daal4py extension build parallelization

INSTALL.md

@@ -172,8 +172,11 @@ The build-process (using setup.py) happens in 4 stages:
 * ``NO_STREAM``: set to '1', 'yes' or alike to build without support for streaming mode
 * ``NO_DPC``: set to '1', 'yes' or alike to build without support of oneDAL DPC++ interfaces
 * ``OFF_ONEDAL_IFACE``: set to '1' to build without the support of oneDAL interfaces
+* ``MAKEFLAGS``: the last `-j` flag determines the number of threads for building the onedal extension. It will default to the number of CPU threads when not set.
 
 **Note:** in order to use distributed mode, `mpi4py` is also required, and needs to be built with the same MPI backend as scikit-learn-intelex.
+**Note:** The `-j` flag in the ``MAKEFLAGS`` environment variable is superseded in `setup.py` modes which support the ``--parallel`` and `-j` command line flags.
+
 
 ### Build Intel(R) Extension for Scikit-learn
 

scripts/build_backend.py

@@ -17,12 +17,10 @@
 # ===============================================================================
 
 import logging
-import multiprocessing
 import os
 import platform as plt
 import subprocess
 import sys
-from math import floor
 from os.path import join as jp
 from sysconfig import get_config_var, get_paths
 
@@ -50,6 +48,7 @@ def custom_build_cmake_clib(
     use_parameters_lib=True,
     use_abs_rpath=False,
     use_gcov=False,
+    n_threads=1,
 ):
     import pybind11
 
@@ -134,18 +133,10 @@ def custom_build_cmake_clib(
     if use_gcov:
         cmake_args += ["-DSKLEARNEX_GCOV=ON"]
 
-    cpu_count = multiprocessing.cpu_count()
-    # limit parallel cmake jobs if memory size is insufficient
-    # TODO: add on all platforms
-    if IS_LIN:
-        with open("/proc/meminfo", "r") as meminfo_file_obj:
-            memfree = meminfo_file_obj.read().split("\n")[1].split(" ")
-            while "" in memfree:
-                memfree.remove("")
-            memfree = int(memfree[1])  # total memory in kB
-        cpu_count = min(cpu_count, floor(max(1, memfree / 2**20)))
-
-    make_args = ["cmake", "--build", abs_build_temp_path, "-j " + str(cpu_count)]
+    # the number of parallel processes is dictated by MAKEFLAGS (see setup.py)
+    # using make conventions (i.e. -j flag) but is set as a cmake argument to
+    # support Windows and Linux simultaneously
+    make_args = ["cmake", "--build", abs_build_temp_path, "-j" + str(n_threads)]
 
     make_install_args = [
         "cmake",

setup.py

@@ -22,10 +22,10 @@
 import os
 import pathlib
 import platform as plt
+import re
 import shutil
 import sys
 import time
-from concurrent.futures import ThreadPoolExecutor
 from ctypes.util import find_library
 from os.path import join as jp
 from sysconfig import get_config_vars
@@ -35,7 +35,6 @@
 import setuptools.command.develop as orig_develop
 from Cython.Build import cythonize
 from setuptools import Extension, setup
-from setuptools.command.build_ext import build_ext as _build_ext
 
 import scripts.build_backend as build_backend
 from scripts.package_helpers import get_packages_with_tests
@@ -52,7 +51,6 @@
 IS_LIN = False
 
 dal_root = os.environ.get("DALROOT")
-n_threads = int(os.environ.get("NTHREADS", os.cpu_count() or 1))
 
 arch_dir = plt.machine()
 plt_dict = {"x86_64": "intel64", "AMD64": "intel64", "aarch64": "arm"}
@@ -335,7 +333,8 @@ def getpyexts():
         library_dirs=ONEDAL_LIBDIRS,
         language="c++",
     )
-    exts.extend(cythonize(ext, nthreads=n_threads))
+
+    exts.extend(cythonize(ext))
 
     if not no_dist:
         mpi_include_dir = include_dir_plat + [np.get_include()] + MPI_INCDIRS
@@ -405,51 +404,48 @@ def get_onedal_py_libs():
     return libs
 
 
-class parallel_build_ext(_build_ext):
-    def build_extensions(self):
-        with ThreadPoolExecutor(max_workers=n_threads) as executor:
-            result_list = [
-                executor.submit(self.build_extension, ext) for ext in self.extensions
-            ]
-        assert all(
-            f.exception() is None for f in result_list
-        ), "There were errors building the extensions"
-
+class onedal_build:
 
-class custom_build:
     def run(self):
+        self.onedal_run()
+        super(onedal_build, self).run()
+        self.onedal_post_build()
+
+    def onedal_run(self):
+        n_threads = self.parallel
+        makeflags = os.getenv("MAKEFLAGS", "")
+        # True is used by setuptools to indicate cpu_count for `parallel`
+        # None is default for setuptools for single threading
+        # take the last defined value in MAKEFLAGS, as it will be the one
+        # used by cmake/make. Do regex in reverse to deal with missing values
+        # and last values simultaneously in a simple fashion
+        regex_inv = r"(?<!\S)\d*(?=j-(?!\S))|$"
+        orig_n_threads = re.findall(regex_inv, makeflags[::-1])[0][::-1]
+
+        if n_threads is None:
+            n_threads = int(orig_n_threads) if orig_n_threads else os.cpu_count() or 1
+        elif n_threads is True:
+            n_threads = os.cpu_count() or 1
+
+        cxx = os.getenv("CXX", "cl" if IS_WIN else "g++")
+        build_onedal = lambda iface: build_backend.custom_build_cmake_clib(
+            iface=iface,
+            cxx=cxx,
+            onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
+            no_dist=no_dist,
+            use_parameters_lib=use_parameters_lib,
+            use_abs_rpath=USE_ABS_RPATH,
+            use_gcov=use_gcov,
+            n_threads=n_threads,
+        )
         if is_onedal_iface:
-            cxx = os.getenv("CXX", "cl" if IS_WIN else "g++")
-            build_backend.custom_build_cmake_clib(
-                iface="host",
-                cxx=cxx,
-                onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
-                no_dist=no_dist,
-                use_parameters_lib=use_parameters_lib,
-                use_abs_rpath=USE_ABS_RPATH,
-                use_gcov=use_gcov,
-            )
-        if dpcpp:
-            if is_onedal_iface:
-                build_backend.custom_build_cmake_clib(
-                    iface="dpc",
-                    onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
-                    no_dist=no_dist,
-                    use_parameters_lib=use_parameters_lib,
-                    use_abs_rpath=USE_ABS_RPATH,
-                    use_gcov=use_gcov,
-                )
+            build_onedal("host")
+            if dpcpp:
+                build_onedal("dpc")
                 if build_distribute:
-                    build_backend.custom_build_cmake_clib(
-                        iface="spmd_dpc",
-                        onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
-                        no_dist=no_dist,
-                        use_parameters_lib=use_parameters_lib,
-                        use_abs_rpath=USE_ABS_RPATH,
-                        use_gcov=use_gcov,
-                    )
-
-    def post_build(self):
+                    build_onedal("spmd_dpc")
+
+    def onedal_post_build(self):
         if IS_MAC:
             import subprocess
 
@@ -472,18 +468,12 @@ def post_build(self):
                         )
 
 
-class develop(orig_develop.develop, custom_build):
-    def run(self):
-        custom_build.run(self)
-        super().run()
-        custom_build.post_build(self)
+class develop(onedal_build, orig_develop.develop):
+    pass
 
 
-class build(orig_build.build, custom_build):
-    def run(self):
-        custom_build.run(self)
-        super().run()
-        custom_build.post_build(self)
+class build(onedal_build, orig_build.build):
+    pass
 
 
 project_urls = {
@@ -581,7 +571,7 @@ def run(self):
     author_email="onedal.maintainers@intel.com",
     maintainer_email="onedal.maintainers@intel.com",
     project_urls=project_urls,
-    cmdclass={"develop": develop, "build": build, "build_ext": parallel_build_ext},
+    cmdclass={"develop": develop, "build": build},
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",