Merge branch 'main' into dev/fix_windows_todo

.ci/pipeline/build-and-test-lnx.yml

@@ -67,20 +67,24 @@ steps:
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
+      if [ -n "${COVERAGE_RCFILE}" ]; then export COVERAGE_FILE=$(pwd)/.coverage.sklearnex; fi
       cd ..
       if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then source ${TBBROOT}/env/vars.sh; fi
       ./s/conda-recipe/run_test.sh --json-report
     env:
       TBBROOT: ${{ variables.TBBROOT }}
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
     displayName: "Sklearnex testing"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then source ${TBBROOT}/env/vars.sh; fi
+      if [ -n "${COVERAGE_RCFILE}" ]; then export COVERAGE_FILE=$(pwd)/.coverage.sklearn; fi
       if [ -z "${NO_DPC}" ]; then export CPU="cpu"; fi
       bash .ci/scripts/run_sklearn_tests.sh $CPU
     env:
       TBBROOT: ${{ variables.TBBROOT }}
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
       NO_DPC: ${{ variables.NO_DPC }}
     displayName: "Sklearn testing"
     condition: succeededOrFailed()

.ci/pipeline/build-and-test-win.yml

@@ -50,14 +50,20 @@ steps:
     displayName: 'Install testing requirements'
   - script: |
       call activate CB
+      if defined COVERAGE_RCFILE set COVERAGE_FILE=%cd%\.coverage.sklearnex
       cd ..
       call s\conda-recipe\run_test.bat s\ --json-report
     displayName: 'Sklearnex testing'
+    env:
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
   - script: |
       call activate CB
+      if defined COVERAGE_RCFILE set COVERAGE_FILE=%cd%\.coverage.sklearn
       bash .ci/scripts/run_sklearn_tests.sh
     displayName: 'Sklearn testing'
     condition: succeededOrFailed()
+    env:
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
   - script: |
       call activate CB
       bash .ci/scripts/run_sklearn_tests.sh

.ci/pipeline/ci.yml

@@ -76,6 +76,8 @@ jobs:
 - job: LinuxCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
+  variables:
+    COVERAGE_RCFILE: '$(Build.Repository.LocalPath)/.coveragerc'
   strategy:
     matrix:
       Python3.9_Sklearn1.0:
@@ -97,9 +99,12 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
   - template: build-and-test-lnx.yml
+  - template: codecov-lnx.yml
 - job: WindowsCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
+  variables:
+    COVERAGE_RCFILE: '$(Build.Repository.LocalPath)\.coveragerc'
   strategy:
     matrix:
       Python3.9_Sklearn1.0:
@@ -121,3 +126,4 @@ jobs:
     vmImage: 'windows-2022'
   steps:
   - template: build-and-test-win.yml
+  - template: codecov-win.yml

.ci/pipeline/codecov-lnx.yml

@@ -0,0 +1,30 @@
+#===============================================================================
+# Copyright contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+steps:
+  - script: |
+      . /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate CB
+      coverage combine .coverage.sklearnex .coverage.sklearn
+      coverage lcov -o coverage.info
+    displayName: "Create coverage report"
+  - script: |
+      curl -Os https://cli.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      export VARARGS="-n azure-${AGENT_OS}-$(PYTHON_VERSION)-$(SKLEARN_VERSION)"
+      ./codecov -v upload-process -Z -t ${CODECOV_TOKEN} "${VARARGS}" -F azure -f coverage.info
+    displayName: "Upload to codecov"
+    env:
+      CODECOV_TOKEN: $(CODECOV_TOKEN)

.ci/pipeline/codecov-win.yml

@@ -0,0 +1,29 @@
+#===============================================================================
+# Copyright contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+steps:
+  - script: |
+      call activate CB
+      coverage combine .coverage.sklearnex .coverage.sklearn
+      coverage lcov -o coverage.info
+    displayName: "Create coverage report"
+  - script: |
+      set PATH=C:\msys64\usr\bin;%PATH%
+      curl -Os https://cli.codecov.io/latest/windows/codecov.exe
+      set VARARGS=-n azure-%AGENT_OS%-$(PYTHON_VERSION)-$(SKLEARN_VERSION)
+      .\codecov.exe -v upload-process -Z -t %CODECOV_TOKEN% %VARARGS% -F azure -f coverage.info
+    displayName: "Upload to codecov"
+    env:
+      CODECOV_TOKEN: $(CODECOV_TOKEN)

.ci/scripts/run_sklearn_tests.py

@@ -58,6 +58,7 @@
         pytest_args += (
             "--cov=onedal",
             "--cov=sklearnex",
+            "--cov-branch",
             f"--cov-config={rc}",
             "--cov-report=",
         )

.github/.codecov.yml

@@ -28,3 +28,9 @@ coverage:
         target: 0
         # Allow for diffs to have no code coverage.
         # threshold: 50
+
+flags:
+  github:
+    after_n_builds: 1
+  azure:
+    after_n_builds: 8

.github/scripts/generate_coverage_reports.sh

@@ -0,0 +1,46 @@
+#===============================================================================
+# Copyright Contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+ci_dir=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
+cd $ci_dir
+
+# create coverage.py report
+coverage combine .coverage.sklearnex .coverage.sklearn
+coverage lcov -o coverage_py_"${1}".info
+
+# create gcov report (lcov format)
+if [[ -n "${SKLEARNEX_GCOV}" ]]; then
+    # extract llvm tool for gcov processing
+    if [[ -z "$2" ]]; then
+        GCOV_EXE="$(dirname $(type -P -a icx))/compiler/llvm-cov gcov"
+    else
+        GCOV_EXE="gcov"
+    fi
+    echo $GCOV_EXE
+    FILTER=$(realpath ./onedal).*
+    echo $FILTER
+
+    NUMPY_TEST=$(python -m pip freeze | grep numpy)
+    # install dependencies
+    # proper operation of gcov with sklearnex requires the header files from
+    # the build numpy, this must be previously set as NUMPY_BUILD
+    python -m pip install gcovr $NUMPY_BUILD
+
+    gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FILTER}" -o coverage_cpp_"${1}".info
+
+    # reinstall previous numpy
+    python -m pip install $NUMPY_TEST
+fi

.github/workflows/ci.yml

@@ -95,6 +95,7 @@ jobs:
           echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
           # enable coverage report generation
           echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV"
+          if [[ -z $DPCFLAG ]]; then echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV"; fi
       - name: apt-get
         run: sudo apt-get update && sudo apt-get install -y clang-format
       - name: dpcpp installation
@@ -111,6 +112,7 @@ jobs:
           source venv/bin/activate
           pip install -r dependencies-dev
           pip list
+          echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV"
       - name: Build daal4py/sklearnex
         run: |
           source venv/bin/activate
@@ -142,13 +144,14 @@ jobs:
       - name: Create coverage report
         run: |
           source venv/bin/activate
-          coverage combine .coverage.sklearnex .coverage.sklearn
-          coverage json -o coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }}
+          bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
       - name: Archive coverage report
         uses: actions/upload-artifact@v4
         with:
           name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
-          path: coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          path: |
+            *_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info
       - name: Sklearn testing [preview]
         run: |
           source venv/bin/activate
@@ -218,7 +221,12 @@ jobs:
           echo C:\msys64\usr\bin;>> %GITHUB_PATH%
           echo NO_DIST=YES>> %GITHUB_ENV%
           set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}"
-          if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%)
+          if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (
+            echo DPCFLAG=>> %GITHUB_OUTPUT%
+            echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV%
+          ) else (
+            echo DPCFLAG="0">> %GITHUB_OUTPUT%
+          )
           echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV%
       - name: Download Intel OpenCL CPU Runtime artifact
         if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
@@ -242,6 +250,7 @@ jobs:
           pip install --upgrade setuptools
           pip install cpufeature clang-format pyyaml
           pip install -r dependencies-dev
+          for /f "delims=" %%c in ('python -m pip freeze ^| grep numpy') do echo NUMPY_BUILD=%%c>> %GITHUB_ENV%
       - name: System info 
         shell: cmd
         run: |
@@ -288,13 +297,14 @@ jobs:
         shell: cmd
         run: |
           call .\venv\Scripts\activate.bat
-          coverage combine .coverage.sklearnex .coverage.sklearn
-          coverage json -o coverage.win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
+          bash .github/scripts/generate_coverage_reports.sh win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
       - name: Archive coverage report
         uses: actions/upload-artifact@v4
         with:
           name: coverage_win_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
-          path: coverage.win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          path: |
+             *_win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info
       - name: Sklearn testing [preview]
         shell: cmd
         run: |

.github/workflows/codecov.yml

@@ -50,8 +50,8 @@ jobs:
           OWNER="${FULL_NAME%/$NAME}"
           if [ "${{ github.repository_owner }}" != "${OWNER}" ]; then BRANCH="${OWNER}:${BRANCH}"; fi
           if [ $(git branch --show-current) != $BRANCH ]; then PR=$(gh pr view $BRANCH --json number -q .number); fi
-          echo uploading $BRANCH
           SHA=${{ github.event.workflow_run.head_sha }}
+          echo uploading $BRANCH $SHA
           VARARGS="-C ${SHA} -n github-${SHA}"
           # if a PR, pass proper information to codecov-cli about the PR number
           if [ -n "${PR}" ]; then VARARGS="${VARARGS}-${PR} -P ${PR}"; fi

INSTALL.md

@@ -157,8 +157,8 @@ The build-process (using setup.py) happens in 4 stages:
 * Numpy
 * cmake and pybind11
 * A C++ compiler with C++11 support
-* Clang-Format
-* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/uxlfoundation/oneDAL) version 2021.1 or later
+* Clang-Format version >=14
+* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/uxlfoundation/oneDAL) version 2021.1 or later, but be mindful that **the oneDAL version must be <= than that of scikit-learn-intelex** (it's backwards compatible but not forwards compatible).
   * You can use the pre-built `dal-devel` conda package from conda-forge channel
 * MPI (optional, needed for distributed mode)
   * You can use the pre-built `impi_rt` and `impi-devel` conda packages from conda-forge channel

conda-recipe/run_test.bat

@@ -34,7 +34,7 @@ if "%PYTHON%"=="python" (
 
 set "PYTEST_ARGS= "
 
-IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-report= %PYTEST_ARGS%")
+IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-branch --cov-report= %PYTEST_ARGS%")
 
 rem Note: execute with argument --json-report as second argument
 rem in order to produce a JSON report under folder '.pytest_reports'.

conda-recipe/run_test.sh

@@ -55,7 +55,7 @@ function generate_pytest_args {
         ARGS+=("--json-report-file=.pytest_reports/$1_report.json")
     fi
     if [ -n "${COVERAGE_RCFILE}" ]; then
-        ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-report=)
+        ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-branch --cov-report=)
     fi
     printf -- "${ARGS[*]}"
 }

doc/sources/quick-start.rst

@@ -319,11 +319,7 @@ Download the Intel AI Tools `here <https://www.intel.com/content/www/us/en/devel
 Release Notes
 -------------------
 
-<<<<<<< HEAD
 See the `Release Notes <https://github.com/uxlfoundation/scikit-learn-intelex/releases>`_ for each version of Intel® Extension for Scikit-learn*.  
-=======
-See the `Release Notes <https://github.com/intel/scikit-learn-intelex/releases>`_ for each version of Intel® Extension for Scikit-learn*.
->>>>>>> e8a9b150 (CI: add `skywalking-eyes` license header check)
 
 System Requirements
 --------------------

scripts/CMakeLists.txt

@@ -29,6 +29,9 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 option(ADD_ONEDAL_RPATH "Adds oneDAL's file paths to the RPATH here" OFF)
 message(STATUS "ADD_ONEDAL_RPATH:" ${ADD_ONEDAL_RPATH})
 
+option(SKLEARNEX_GCOV "Compile with gcov" OFF)
+message(STATUS "SKLEARNEX_GCOV:" ${SKLEARNEX_GCOV})
+
 if(WIN32)
     # hint CMake to get python from PYTHON env. variable if defined
     if(DEFINED ENV{PYTHON})
@@ -122,6 +125,23 @@ if(IFACE STREQUAL "host")
         endif()
     endif()
 
+    if(SKLEARNEX_GCOV)
+        if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+            if(WIN32)
+                set(CMAKE_CXX_FLAGS "/clang:--coverage ${CMAKE_CXX_FLAGS}")
+                list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib")
+            else()
+                set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}")
+                set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}")
+            endif()
+        elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+            set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}")
+            set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}")
+        else()
+            message(WARNING "Code coverage will not be generated for target: host")
+        endif()
+    endif()
+
     list(APPEND COMPILE_DEFINITIONS "NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION")
 
 elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC)
@@ -137,11 +157,12 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC)
         set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF)
     endif()
 
-    if(CMAKE_CXX_COMPILER MATCHES ".*icpx" OR CMAKE_CXX_COMPILER MATCHES ".*icx")
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
         set(CMAKE_CXX_FLAGS "-fsycl ${CMAKE_CXX_FLAGS}")
+
     endif()
 
-    if(CMAKE_C_COMPILER MATCHES ".*icpx" OR CMAKE_C_COMPILER MATCHES ".*icx")
+    if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM")
         set(CMAKE_C_FLAGS "-fsycl ${CMAKE_C_FLAGS}")
     endif()
 
@@ -169,6 +190,18 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC)
         endif()
     endif()
 
+    if(SKLEARNEX_GCOV)
+        if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32)
+            set(CMAKE_CXX_FLAGS "/clang:-Xarch_host /clang:--coverage ${CMAKE_CXX_FLAGS}")
+            list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib")
+        elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND UNIX)
+            set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}")
+            set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}")
+        else()
+            message(WARNING "Code coverage will not be generated for target: " ${IFACE})
+        endif()
+    endif()
+
     if(IFACE_IS_SPMD_DPC)
         set(MPI_LIBRARY ${MPI_LIBS})
     endif()

scripts/build_backend.py

@@ -47,6 +47,7 @@ def custom_build_cmake_clib(
     no_dist=True,
     use_parameters_lib=True,
     use_abs_rpath=False,
+    use_gcov=False,
 ):
     import pybind11