Merge branch 'uxlfoundation:main' into dev/eglaser-online-spmd-refactor

.ci/pipeline/build-and-test-lnx.yml

@@ -67,20 +67,24 @@ steps:
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
+      if [ -n "${COVERAGE_RCFILE}" ]; then export COVERAGE_FILE=$(pwd)/.coverage.sklearnex; fi
       cd ..
       if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then source ${TBBROOT}/env/vars.sh; fi
       ./s/conda-recipe/run_test.sh --json-report
     env:
       TBBROOT: ${{ variables.TBBROOT }}
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
     displayName: "Sklearnex testing"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       if [ -n "${TBBROOT}" ] && [ "${TBBROOT}" != "${CONDA_PREFIX}" ]; then source ${TBBROOT}/env/vars.sh; fi
+      if [ -n "${COVERAGE_RCFILE}" ]; then export COVERAGE_FILE=$(pwd)/.coverage.sklearn; fi
       if [ -z "${NO_DPC}" ]; then export CPU="cpu"; fi
       bash .ci/scripts/run_sklearn_tests.sh $CPU
     env:
       TBBROOT: ${{ variables.TBBROOT }}
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
       NO_DPC: ${{ variables.NO_DPC }}
     displayName: "Sklearn testing"
     condition: succeededOrFailed()

.ci/pipeline/build-and-test-win.yml

@@ -50,14 +50,20 @@ steps:
     displayName: 'Install testing requirements'
   - script: |
       call activate CB
+      if defined COVERAGE_RCFILE set COVERAGE_FILE=%cd%\.coverage.sklearnex
       cd ..
       call s\conda-recipe\run_test.bat s\ --json-report
     displayName: 'Sklearnex testing'
+    env:
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
   - script: |
       call activate CB
+      if defined COVERAGE_RCFILE set COVERAGE_FILE=%cd%\.coverage.sklearn
       bash .ci/scripts/run_sklearn_tests.sh
     displayName: 'Sklearn testing'
     condition: succeededOrFailed()
+    env:
+      COVERAGE_RCFILE: ${{ variables.COVERAGE_RCFILE }}
   - script: |
       call activate CB
       bash .ci/scripts/run_sklearn_tests.sh

.ci/pipeline/ci.yml

@@ -76,6 +76,8 @@ jobs:
 - job: LinuxCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
+  variables:
+    COVERAGE_RCFILE: '$(Build.Repository.LocalPath)/.coveragerc'
   strategy:
     matrix:
       Python3.9_Sklearn1.0:
@@ -97,9 +99,12 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
   - template: build-and-test-lnx.yml
+  - template: codecov-lnx.yml
 - job: WindowsCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
+  variables:
+    COVERAGE_RCFILE: '$(Build.Repository.LocalPath)\.coveragerc'
   strategy:
     matrix:
       Python3.9_Sklearn1.0:
@@ -121,3 +126,4 @@ jobs:
     vmImage: 'windows-2022'
   steps:
   - template: build-and-test-win.yml
+  - template: codecov-win.yml

.ci/pipeline/codecov-lnx.yml

@@ -0,0 +1,30 @@
+#===============================================================================
+# Copyright contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+steps:
+  - script: |
+      . /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate CB
+      coverage combine .coverage.sklearnex .coverage.sklearn
+      coverage lcov -o coverage.info
+    displayName: "Create coverage report"
+  - script: |
+      curl -Os https://cli.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      export VARARGS="-n azure-${AGENT_OS}-$(PYTHON_VERSION)-$(SKLEARN_VERSION)"
+      ./codecov -v upload-process -Z -t ${CODECOV_TOKEN} "${VARARGS}" -F azure -f coverage.info
+    displayName: "Upload to codecov"
+    env:
+      CODECOV_TOKEN: $(CODECOV_TOKEN)

.ci/pipeline/codecov-win.yml

@@ -0,0 +1,29 @@
+#===============================================================================
+# Copyright contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+steps:
+  - script: |
+      call activate CB
+      coverage combine .coverage.sklearnex .coverage.sklearn
+      coverage lcov -o coverage.info
+    displayName: "Create coverage report"
+  - script: |
+      set PATH=C:\msys64\usr\bin;%PATH%
+      curl -Os https://cli.codecov.io/latest/windows/codecov.exe
+      set VARARGS=-n azure-%AGENT_OS%-$(PYTHON_VERSION)-$(SKLEARN_VERSION)
+      .\codecov.exe -v upload-process -Z -t %CODECOV_TOKEN% %VARARGS% -F azure -f coverage.info
+    displayName: "Upload to codecov"
+    env:
+      CODECOV_TOKEN: $(CODECOV_TOKEN)

.ci/scripts/run_sklearn_tests.py

@@ -58,6 +58,7 @@
         pytest_args += (
             "--cov=onedal",
             "--cov=sklearnex",
+            "--cov-branch",
             f"--cov-config={rc}",
             "--cov-report=",
         )

.clang-format

@@ -0,0 +1,3 @@
+---
+DisableFormat: true
+...

.github/.codecov.yml

@@ -28,3 +28,9 @@ coverage:
         target: 0
         # Allow for diffs to have no code coverage.
         # threshold: 50
+
+flags:
+  github:
+    after_n_builds: 1
+  azure:
+    after_n_builds: 8

.github/CODEOWNERS

@@ -1,25 +1,27 @@
 # Infra and tools
+.*                            @napetrov @Alexsandruss @homksei @ahuber21 @ethanglaser
 .ci/                          @napetrov @Alexsandruss @homksei @ahuber21 @ethanglaser
-.github/                      @napetrov @Alexsandruss @homksei @ahuber21 @ethanglaser @samir-nasibli @icfaust
 .circleci/                    @napetrov @Alexsandruss @homksei @ahuber21 @ethanglaser
+.github/                      @napetrov @Alexsandruss @homksei @ahuber21 @ethanglaser @samir-nasibli @icfaust
 
 # Docs
 *.md                          @maria-Petrova @napetrov @Alexsandruss @samir-nasibli @icfaust @david-cortes-intel
 doc/                          @maria-Petrova @napetrov @Alexsandruss @samir-nasibli @icfaust @david-cortes-intel
 requirements-doc.txt          @Alexsandruss @samir-nasibli @icfaust @david-cortes-intel
 
-# DPC++
+# sklearnex
 onedal/                       @Alexsandruss @samir-nasibli @icfaust
 sklearnex/                    @Alexsandruss @samir-nasibli @icfaust
 
 # Examples
 examples/                     @maria-Petrova @Alexsandruss @samir-nasibli @napetrov
 
 # Dependencies
-setup.py                      @napetrov @Alexsandruss @samir-nasibli @icfaust
-requirements*                 @napetrov @Alexsandruss @samir-nasibli @homksei @ahuber21 @ethanglaser
-dependencies-dev              @napetrov @Alexsandruss @samir-nasibli @homksei @ahuber21 @ethanglaser
 conda-recipe/                 @napetrov @Alexsandruss
+dependencies-dev              @napetrov @Alexsandruss @samir-nasibli @homksei @ahuber21 @ethanglaser
+pyproject.toml                @napetrov @Alexsandruss @samir-nasibli @homksei @ahuber21 @ethanglaser
+requirements*                 @napetrov @Alexsandruss @samir-nasibli @homksei @ahuber21 @ethanglaser
+setup.*                       @napetrov @Alexsandruss @samir-nasibli @icfaust
 
 # Model builders
 *model_builders*              @razdoburdin @ahuber21 @avolkov-intel
@@ -29,13 +31,16 @@ conda-recipe/                 @napetrov @Alexsandruss
 
 # Testing
 **/test*.py                   @Alexsandruss @samir-nasibli @icfaust
+deselected_tests.yaml         @Alexsandruss @samir-nasibli @icfaust
+tests/                        @Alexsandruss @samir-nasibli @icfaust
 
 # Distributed
 *spmd*                        @samir-nasibli @ethanglaser
 
-# Scikit-learn patching
+# daal4py
 daal4py/sklearn/              @Alexsandruss @samir-nasibli @icfaust
 
 # Core
-src/                          @Alexsandruss @samir-nasibli @icfaust
 generator/                    @Alexsandruss @samir-nasibli @icfaust
+scripts/                      @Alexsandruss @samir-nasibli @icfaust
+src/                          @Alexsandruss @samir-nasibli @icfaust

.github/scripts/generate_coverage_reports.sh

@@ -0,0 +1,46 @@
+#===============================================================================
+# Copyright Contributors to the oneDAL project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+ci_dir=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))
+cd $ci_dir
+
+# create coverage.py report
+coverage combine .coverage.sklearnex .coverage.sklearn
+coverage lcov -o coverage_py_"${1}".info
+
+# create gcov report (lcov format)
+if [[ -n "${SKLEARNEX_GCOV}" ]]; then
+    # extract llvm tool for gcov processing
+    if [[ -z "$2" ]]; then
+        GCOV_EXE="$(dirname $(type -P -a icx))/compiler/llvm-cov gcov"
+    else
+        GCOV_EXE="gcov"
+    fi
+    echo $GCOV_EXE
+    FILTER=$(realpath ./onedal).*
+    echo $FILTER
+
+    NUMPY_TEST=$(python -m pip freeze | grep numpy)
+    # install dependencies
+    # proper operation of gcov with sklearnex requires the header files from
+    # the build numpy, this must be previously set as NUMPY_BUILD
+    python -m pip install gcovr $NUMPY_BUILD
+
+    gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FILTER}" -o coverage_cpp_"${1}".info
+
+    # reinstall previous numpy
+    python -m pip install $NUMPY_TEST
+fi

.github/workflows/ci.yml

@@ -95,6 +95,7 @@ jobs:
           echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
           # enable coverage report generation
           echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV"
+          if [[ -z $DPCFLAG ]]; then echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV"; fi
       - name: apt-get
         run: sudo apt-get update && sudo apt-get install -y clang-format
       - name: dpcpp installation
@@ -111,6 +112,7 @@ jobs:
           source venv/bin/activate
           pip install -r dependencies-dev
           pip list
+          echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV"
       - name: Build daal4py/sklearnex
         run: |
           source venv/bin/activate
@@ -142,13 +144,14 @@ jobs:
       - name: Create coverage report
         run: |
           source venv/bin/activate
-          coverage combine .coverage.sklearnex .coverage.sklearn
-          coverage json -o coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }}
+          bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
       - name: Archive coverage report
         uses: actions/upload-artifact@v4
         with:
           name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
-          path: coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          path: |
+            *_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info
       - name: Sklearn testing [preview]
         run: |
           source venv/bin/activate
@@ -218,7 +221,12 @@ jobs:
           echo C:\msys64\usr\bin;>> %GITHUB_PATH%
           echo NO_DIST=YES>> %GITHUB_ENV%
           set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}"
-          if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%)
+          if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (
+            echo DPCFLAG=>> %GITHUB_OUTPUT%
+            echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV%
+          ) else (
+            echo DPCFLAG="0">> %GITHUB_OUTPUT%
+          )
           echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV%
       - name: Download Intel OpenCL CPU Runtime artifact
         if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
@@ -242,6 +250,7 @@ jobs:
           pip install --upgrade setuptools
           pip install cpufeature clang-format pyyaml
           pip install -r dependencies-dev
+          for /f "delims=" %%c in ('python -m pip freeze ^| grep numpy') do echo NUMPY_BUILD=%%c>> %GITHUB_ENV%
       - name: System info 
         shell: cmd
         run: |
@@ -288,13 +297,14 @@ jobs:
         shell: cmd
         run: |
           call .\venv\Scripts\activate.bat
-          coverage combine .coverage.sklearnex .coverage.sklearn
-          coverage json -o coverage.win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
+          bash .github/scripts/generate_coverage_reports.sh win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
       - name: Archive coverage report
         uses: actions/upload-artifact@v4
         with:
           name: coverage_win_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
-          path: coverage.win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json
+          path: |
+             *_win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info
       - name: Sklearn testing [preview]
         shell: cmd
         run: |

.github/workflows/codecov.yml

@@ -50,8 +50,8 @@ jobs:
           OWNER="${FULL_NAME%/$NAME}"
           if [ "${{ github.repository_owner }}" != "${OWNER}" ]; then BRANCH="${OWNER}:${BRANCH}"; fi
           if [ $(git branch --show-current) != $BRANCH ]; then PR=$(gh pr view $BRANCH --json number -q .number); fi
-          echo uploading $BRANCH
           SHA=${{ github.event.workflow_run.head_sha }}
+          echo uploading $BRANCH $SHA
           VARARGS="-C ${SHA} -n github-${SHA}"
           # if a PR, pass proper information to codecov-cli about the PR number
           if [ -n "${PR}" ]; then VARARGS="${VARARGS}-${PR} -P ${PR}"; fi

.pre-commit-config.yaml

@@ -25,3 +25,7 @@ repos:
     hooks:
       - id: isort
         language_version: python3.10
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v14.0.6
+    hooks:
+      - id: clang-format

INSTALL.md

@@ -157,8 +157,8 @@ The build-process (using setup.py) happens in 4 stages:
 * Numpy
 * cmake and pybind11
 * A C++ compiler with C++11 support
-* Clang-Format
-* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/uxlfoundation/oneDAL) version 2021.1 or later
+* Clang-Format version >=14
+* [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/uxlfoundation/oneDAL) version 2021.1 or later, but be mindful that **the oneDAL version must be <= than that of scikit-learn-intelex** (it's backwards compatible but not forwards compatible).
   * You can use the pre-built `dal-devel` conda package from conda-forge channel
 * MPI (optional, needed for distributed mode)
   * You can use the pre-built `impi_rt` and `impi-devel` conda packages from conda-forge channel

conda-recipe/run_test.bat

@@ -34,7 +34,7 @@ if "%PYTHON%"=="python" (
 
 set "PYTEST_ARGS= "
 
-IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-report= %PYTEST_ARGS%")
+IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-branch --cov-report= %PYTEST_ARGS%")
 
 rem Note: execute with argument --json-report as second argument
 rem in order to produce a JSON report under folder '.pytest_reports'.

conda-recipe/run_test.sh

@@ -55,7 +55,7 @@ function generate_pytest_args {
         ARGS+=("--json-report-file=.pytest_reports/$1_report.json")
     fi
     if [ -n "${COVERAGE_RCFILE}" ]; then
-        ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-report=)
+        ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-branch --cov-report=)
     fi
     printf -- "${ARGS[*]}"
 }

dependencies-dev

@@ -3,5 +3,5 @@ Jinja2==3.1.5
 numpy==2.0.1 ; python_version <= '3.9'
 numpy==2.2.1 ; python_version > '3.9'
 pybind11==2.13.6
-cmake==3.31.2
-setuptools==75.7.0
+cmake==3.31.4
+setuptools==75.8.0

doc/sources/quick-start.rst

@@ -319,11 +319,7 @@ Download the Intel AI Tools `here <https://www.intel.com/content/www/us/en/devel
 Release Notes
 -------------------
 
-<<<<<<< HEAD
 See the `Release Notes <https://github.com/uxlfoundation/scikit-learn-intelex/releases>`_ for each version of Intel® Extension for Scikit-learn*.  
-=======
-See the `Release Notes <https://github.com/intel/scikit-learn-intelex/releases>`_ for each version of Intel® Extension for Scikit-learn*.
->>>>>>> e8a9b150 (CI: add `skywalking-eyes` license header check)
 
 System Requirements
 --------------------