From 2899fb385ec0a58f9a5aeadeebd2123144746186 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 9 Jan 2025 09:50:23 +0100 Subject: [PATCH 01/72] Update ci.yml --- .ci/pipeline/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 3e761c79bc..4aa030e3f4 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -48,6 +48,10 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: + - script: + echo $TEST_VAR + env: + TEST_VAR: $(CODECOV_TOKEN) - template: linting.yml - job: LinuxCondaRecipe dependsOn: Lint From 1b2302b2e2a70a59c018e15ce838b7fdeb3d12d9 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 9 Jan 2025 09:55:28 +0100 Subject: [PATCH 02/72] Update ci.yml --- .ci/pipeline/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 4aa030e3f4..3e761c79bc 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -48,10 +48,6 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: - - script: - echo $TEST_VAR - env: - TEST_VAR: $(CODECOV_TOKEN) - template: linting.yml - job: LinuxCondaRecipe dependsOn: Lint From ec3953e16143f900beaa02c3e937ceeef7b4c7b6 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 9 Jan 2025 13:42:27 +0100 Subject: [PATCH 03/72] Update ci.yml --- .ci/pipeline/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 3e761c79bc..b9d22a1277 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -60,6 +60,8 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: + - script: | + echo $(CODECOV_TOKEN_OPEN) - template: conda-recipe-lnx.yml - job: WindowsCondaRecipe dependsOn: Lint From 356313349b24557d6107969a85fe3b4f5f87a1dd Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 9 Jan 2025 14:18:28 +0100 Subject: [PATCH 04/72] Update ci.yml --- .ci/pipeline/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index b9d22a1277..fd7efb44ae 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -48,6 +48,8 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: + - script: | + echo $(CODECOV_TOKEN_OPEN) - template: linting.yml - job: LinuxCondaRecipe dependsOn: Lint @@ -60,8 +62,6 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: - - script: | - echo $(CODECOV_TOKEN_OPEN) - template: conda-recipe-lnx.yml - job: WindowsCondaRecipe dependsOn: Lint From ad933e2d80811696549b8d9a86441938f486b5e3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Thu, 9 Jan 2025 14:20:45 +0100 Subject: [PATCH 05/72] Update ci.yml --- .ci/pipeline/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index fd7efb44ae..356ed72b08 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -49,7 +49,7 @@ jobs: vmImage: 'ubuntu-22.04' steps: - script: | - echo $(CODECOV_TOKEN_OPEN) + echo $(CODECOV_TOKEN) - template: linting.yml - job: LinuxCondaRecipe dependsOn: Lint From 6f0ed1271482ba0b4f648a341489e6cb07511846 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 01:56:33 +0100 Subject: [PATCH 06/72] force use of coverage to start --- scripts/CMakeLists.txt | 14 +++++++++++++- scripts/build_backend.py | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index ed6a781b76..2f4e1e0973 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -29,6 +29,9 @@ set(CMAKE_CXX_EXTENSIONS OFF) option(ADD_ONEDAL_RPATH "Adds oneDAL's file paths to the RPATH here" OFF) message(STATUS "ADD_ONEDAL_RPATH:" ${ADD_ONEDAL_RPATH}) +option(ONEDAL_GCOV "Compile with gcov" OFF) +message(STATUS "ONEDAL_GCOV:" ${ONEDAL_GCOV}) + if(WIN32) # hint CMake to get python from PYTHON env. variable if defined if(DEFINED ENV{PYTHON}) @@ -60,6 +63,15 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") set(PYTHON_ONEDAL "${CMAKE_CURRENT_SOURCE_DIR}/../onedal/") +if(ONEDAL_GCOV) + if(CMAKE_CXX_COMPILER MATCHES ".*icpx" OR CMAKE_CXX_COMPILER MATCHES ".*icx" OR CMAKE_CXX_COMPILER MATCHES ".*g++") + set(CMAKE_CXX_FLAGS "--coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(FATAL_ERROR "Unsupported compiler for code coverage.") + endif() +endif() + file(GLOB_RECURSE sources ${PYTHON_ONEDAL}/**/*.cpp ${PYTHON_ONEDAL}/*.cpp @@ -143,7 +155,7 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) if(CMAKE_C_COMPILER MATCHES ".*icpx" OR CMAKE_C_COMPILER MATCHES ".*icx") set(CMAKE_C_FLAGS "-fsycl ${CMAKE_C_FLAGS}") - endif() + endif() set(ONEDAL_LIBRARIES "") if(WIN32) diff --git a/scripts/build_backend.py b/scripts/build_backend.py index f3546f480e..cf14a9648f 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -130,6 +130,9 @@ def custom_build_cmake_clib( if use_abs_rpath: cmake_args += ["-DADD_ONEDAL_RPATH=ON"] + if True: + cmake_args += ["-DONEDAL_GCOV=ON"] + cpu_count = multiprocessing.cpu_count() # limit parallel cmake jobs if memory size is insufficient # TODO: add on all platforms From 8915d244b6e6a7209f018804d04bbbf04b0a4191 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 02:08:26 +0100 Subject: [PATCH 07/72] try again --- scripts/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 2f4e1e0973..07c7709896 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -64,7 +64,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") set(PYTHON_ONEDAL "${CMAKE_CURRENT_SOURCE_DIR}/../onedal/") if(ONEDAL_GCOV) - if(CMAKE_CXX_COMPILER MATCHES ".*icpx" OR CMAKE_CXX_COMPILER MATCHES ".*icx" OR CMAKE_CXX_COMPILER MATCHES ".*g++") + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "--coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "--coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") else() @@ -149,11 +149,11 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) endif() - if(CMAKE_CXX_COMPILER MATCHES ".*icpx" OR CMAKE_CXX_COMPILER MATCHES ".*icx") + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_CXX_FLAGS "-fsycl ${CMAKE_CXX_FLAGS}") endif() - if(CMAKE_C_COMPILER MATCHES ".*icpx" OR CMAKE_C_COMPILER MATCHES ".*icx") + if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_C_FLAGS "-fsycl ${CMAKE_C_FLAGS}") endif() From cd4f95e830884e3084cd7cbb70b90dd02fd25d0d Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 02:48:31 +0100 Subject: [PATCH 08/72] attempt again --- scripts/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 07c7709896..609e6e5081 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -65,8 +65,8 @@ set(PYTHON_ONEDAL "${CMAKE_CURRENT_SOURCE_DIR}/../onedal/") if(ONEDAL_GCOV) if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "--coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "--coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_CXX_FLAGS "-fprofile-arcs -ftest-coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-lgcov -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") else() message(FATAL_ERROR "Unsupported compiler for code coverage.") endif() From 765fa20cc5766449aa756d2aee5053583b5a6a40 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 02:57:12 +0100 Subject: [PATCH 09/72] add -Xarch_host --- scripts/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 609e6e5081..464bdc8cb7 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -65,8 +65,8 @@ set(PYTHON_ONEDAL "${CMAKE_CURRENT_SOURCE_DIR}/../onedal/") if(ONEDAL_GCOV) if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "-fprofile-arcs -ftest-coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-lgcov -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_CXX_FLAGS "-Xarch_host -fprofile-arcs -Xarch_host -ftest-coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-lgcov -Xarch_host -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") else() message(FATAL_ERROR "Unsupported compiler for code coverage.") endif() From e5ca42d7ea765f4202deb7eebf145a1576b381b1 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 03:01:01 +0100 Subject: [PATCH 10/72] further checking --- scripts/CMakeLists.txt | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 464bdc8cb7..f8af45c2ed 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -63,14 +63,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") set(PYTHON_ONEDAL "${CMAKE_CURRENT_SOURCE_DIR}/../onedal/") -if(ONEDAL_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "-Xarch_host -fprofile-arcs -Xarch_host -ftest-coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-lgcov -Xarch_host -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") - else() - message(FATAL_ERROR "Unsupported compiler for code coverage.") - endif() -endif() + file(GLOB_RECURSE sources ${PYTHON_ONEDAL}/**/*.cpp @@ -149,6 +142,16 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) endif() + if(ONEDAL_GCOV) + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "-Xarch_host -fprofile-arcs -Xarch_host -ftest-coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-lgcov -Xarch_host -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(FATAL_ERROR "Unsupported compiler for code coverage.") + endif() + endif() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_CXX_FLAGS "-fsycl ${CMAKE_CXX_FLAGS}") endif() From ca44fb7cf3abab2d255e5292d6ac455f1b7df0dc Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 04:06:26 +0100 Subject: [PATCH 11/72] make verbose --- scripts/CMakeLists.txt | 4 ++-- scripts/build_backend.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index f8af45c2ed..d1ef8d3b32 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -144,8 +144,8 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) if(ONEDAL_GCOV) if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "-Xarch_host -fprofile-arcs -Xarch_host -ftest-coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-lgcov -Xarch_host -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_CXX_FLAGS "-Xarch_host --coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") else() message(FATAL_ERROR "Unsupported compiler for code coverage.") endif() diff --git a/scripts/build_backend.py b/scripts/build_backend.py index cf14a9648f..e11bb4f39e 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -118,6 +118,7 @@ def custom_build_cmake_clib( "-DoneDAL_LIBRARY_DIR=" + jp(os.environ["DALROOT"], "lib", arch_dir), "-Dpybind11_DIR=" + pybind11.get_cmake_dir(), "-DoneDAL_USE_PARAMETERS_LIB=" + use_parameters_arg, + "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", ] if build_distribute: From adfa45586e54ea4efd3c8b5da90efe14c6066f48 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 14:47:34 +0100 Subject: [PATCH 12/72] attempts at integration --- .github/scripts/generate_coverage_reports.sh | 34 ++++++++++++++++++++ .github/workflows/ci.yml | 15 +++++---- scripts/CMakeLists.txt | 21 ++++++------ scripts/build_backend.py | 3 +- setup.py | 4 +++ 5 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 .github/scripts/generate_coverage_reports.sh diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh new file mode 100644 index 0000000000..332f464475 --- /dev/null +++ b/.github/scripts/generate_coverage_reports.sh @@ -0,0 +1,34 @@ +#=============================================================================== +# Copyright Contributors to the oneDAL project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +# create coverage.py report +coverage combine .coverage.sklearnex .coverage.sklearn +coverage json -o coverage."${1}".json + +# create gcov report (lcov format) +if [[ $OSTYPE == *"linux"* ]]; then + # extract llvm tool for gcov processing + if [[ -n "$2" ]]; then + GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" + else + GCOV_EXE="gcov" + fi + + FILTER=$(dirname $(realpath .)).* + gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o coverage"${1}".info + sed -i "s|${PWD}/||g" coverage"${1}".info + # remove absolute filepath to match coverage.py file +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9450f922ae..15c72b9a15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,7 @@ jobs: echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT" # enable coverage report generation echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" + echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV" - name: apt-get run: sudo apt-get update && sudo apt-get install -y clang-format - name: dpcpp installation @@ -139,16 +140,19 @@ jobs: export COVERAGE_FILE=$(pwd)/.coverage.sklearn if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi bash .ci/scripts/run_sklearn_tests.sh $CPU - - name: Create coverage report + - name: Create coverage reports run: | source venv/bin/activate - coverage combine .coverage.sklearnex .coverage.sklearn - coverage json -o coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json + source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} + pip install gcovr + bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }} - name: Archive coverage report uses: actions/upload-artifact@v4 with: name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} - path: coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json + path: | + coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info + coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json - name: Sklearn testing [preview] run: | source venv/bin/activate @@ -288,8 +292,7 @@ jobs: shell: cmd run: | call .\venv\Scripts\activate.bat - coverage combine .coverage.sklearnex .coverage.sklearn - coverage json -o coverage.win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json + bash .github/scripts/generate_coverage_reports.sh win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} - name: Archive coverage report uses: actions/upload-artifact@v4 with: diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index d1ef8d3b32..b39f948005 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -57,14 +57,21 @@ else() message(FATAL_ERROR "Unsupported system.") endif() +if(ONEDAL_GCOV) + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(FATAL_ERROR "Unsupported compiler for code coverage.") + endif() +endif() + set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") set(PYTHON_ONEDAL "${CMAKE_CURRENT_SOURCE_DIR}/../onedal/") - - file(GLOB_RECURSE sources ${PYTHON_ONEDAL}/**/*.cpp ${PYTHON_ONEDAL}/*.cpp @@ -143,15 +150,11 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) endif() if(ONEDAL_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "-Xarch_host --coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage -fprofile-dir=${PYTHON_ONEDAL} ${CMAKE_SHARED_LINKER_FLAGS}") - else() - message(FATAL_ERROR "Unsupported compiler for code coverage.") - endif() + #assumes first argument is --coverage, allowing host code profiling + set(CMAKE_CXX_FLAGS "-Xarch_host ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host ${CMAKE_SHARED_LINKER_FLAGS}") endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_CXX_FLAGS "-fsycl ${CMAKE_CXX_FLAGS}") endif() diff --git a/scripts/build_backend.py b/scripts/build_backend.py index e11bb4f39e..a1fb716de7 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -49,6 +49,7 @@ def custom_build_cmake_clib( no_dist=True, use_parameters_lib=True, use_abs_rpath=False, + use_gcov=False, ): import pybind11 @@ -131,7 +132,7 @@ def custom_build_cmake_clib( if use_abs_rpath: cmake_args += ["-DADD_ONEDAL_RPATH=ON"] - if True: + if use_gcov: cmake_args += ["-DONEDAL_GCOV=ON"] cpu_count = multiprocessing.cpu_count() diff --git a/setup.py b/setup.py index 8166f00d07..ed48c226e3 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ no_dist = True if "NO_DIST" in os.environ and os.environ["NO_DIST"] in trues else False no_dpc = True if "NO_DPC" in os.environ and os.environ["NO_DPC"] in trues else False no_stream = "NO_STREAM" in os.environ and os.environ["NO_STREAM"] in trues +use_gcov = "SKLEARNEX_GCOV" in os.environ and os.environ["SKLEARNEX_GCOV"] in trues debug_build = os.getenv("DEBUG_BUILD") == "1" mpi_root = None if no_dist else os.environ["MPIROOT"] dpcpp = ( @@ -426,6 +427,7 @@ def run(self): no_dist=no_dist, use_parameters_lib=use_parameters_lib, use_abs_rpath=USE_ABS_RPATH, + use_gcov=use_gcov, ) if dpcpp: if is_onedal_iface: @@ -435,6 +437,7 @@ def run(self): no_dist=no_dist, use_parameters_lib=use_parameters_lib, use_abs_rpath=USE_ABS_RPATH, + use_gcov=use_gcov, ) if build_distribute: build_backend.custom_build_cmake_clib( @@ -443,6 +446,7 @@ def run(self): no_dist=no_dist, use_parameters_lib=use_parameters_lib, use_abs_rpath=USE_ABS_RPATH, + use_gcov=use_gcov, ) def post_build(self): From 228f6e3f93361319f8e567eabd8b4634eee9aa07 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 15:24:02 +0100 Subject: [PATCH 13/72] add echos for testing --- .ci/pipeline/ci.yml | 2 -- .github/scripts/generate_coverage_reports.sh | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 356ed72b08..3e761c79bc 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -48,8 +48,6 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: - - script: | - echo $(CODECOV_TOKEN) - template: linting.yml - job: LinuxCondaRecipe dependsOn: Lint diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 332f464475..27707b5048 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -21,13 +21,14 @@ coverage json -o coverage."${1}".json # create gcov report (lcov format) if [[ $OSTYPE == *"linux"* ]]; then # extract llvm tool for gcov processing - if [[ -n "$2" ]]; then + if [[ -z "$2" ]]; then GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" else GCOV_EXE="gcov" fi - + echo $GCOV_EXE FILTER=$(dirname $(realpath .)).* + echo $FILTER gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o coverage"${1}".info sed -i "s|${PWD}/||g" coverage"${1}".info # remove absolute filepath to match coverage.py file From 9a6a7041ed7368b95d2da41abe02b43d90842d5e Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 15:25:02 +0100 Subject: [PATCH 14/72] move installation --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15c72b9a15..7bf53466d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,7 @@ jobs: source .github/scripts/activate_components.sh bash .ci/scripts/setup_sklearn.sh ${{ matrix.SKLEARN_VERSION }} pip install --upgrade -r requirements-test.txt - pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml + pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml gcovr if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then pip install dpctl==${{ env.DPCTL_VERSION }} dpnp==${{ env.DPNP_VERSION }}; fi pip list - name: Sklearnex testing @@ -144,7 +144,6 @@ jobs: run: | source venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} - pip install gcovr bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }} - name: Archive coverage report uses: actions/upload-artifact@v4 From b22c8a935e8dd3e5d805606ccc48fedf484943be Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Fri, 10 Jan 2025 16:01:29 +0100 Subject: [PATCH 15/72] ignore error induced by numpy switch --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 27707b5048..701867e0bb 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -29,7 +29,7 @@ if [[ $OSTYPE == *"linux"* ]]; then echo $GCOV_EXE FILTER=$(dirname $(realpath .)).* echo $FILTER - gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o coverage"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o coverage"${1}".info --gcov-ignore-errors=no_working_dir_found sed -i "s|${PWD}/||g" coverage"${1}".info # remove absolute filepath to match coverage.py file fi From f1a229946f837245793bff8081f55e94119bbcba Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 10 Jan 2025 19:27:49 +0100 Subject: [PATCH 16/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 701867e0bb..bbf67ed580 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -29,7 +29,7 @@ if [[ $OSTYPE == *"linux"* ]]; then echo $GCOV_EXE FILTER=$(dirname $(realpath .)).* echo $FILTER - gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o coverage"${1}".info --gcov-ignore-errors=no_working_dir_found + gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov -v --filter "${FITLER}" -o coverage"${1}".info --gcov-ignore-errors=no_working_dir_found sed -i "s|${PWD}/||g" coverage"${1}".info # remove absolute filepath to match coverage.py file fi From 7db1eb57da69d6f3292d87a4f6eb320266affa0a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 10 Jan 2025 20:48:49 +0100 Subject: [PATCH 17/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index bbf67ed580..77cf58fe14 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -27,9 +27,9 @@ if [[ $OSTYPE == *"linux"* ]]; then GCOV_EXE="gcov" fi echo $GCOV_EXE - FILTER=$(dirname $(realpath .)).* + FILTER=$(dirname $(realpath ./onedal)).* echo $FILTER - gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov -v --filter "${FITLER}" -o coverage"${1}".info --gcov-ignore-errors=no_working_dir_found + gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov -v --filter "${FITLER}" -o coverage"${1}".info sed -i "s|${PWD}/||g" coverage"${1}".info # remove absolute filepath to match coverage.py file fi From 6bbba032ffbf2e60afbf8388a5ac07e67c4b18be Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 10 Jan 2025 22:10:39 +0100 Subject: [PATCH 18/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 77cf58fe14..3fa3e115c7 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -27,7 +27,7 @@ if [[ $OSTYPE == *"linux"* ]]; then GCOV_EXE="gcov" fi echo $GCOV_EXE - FILTER=$(dirname $(realpath ./onedal)).* + FILTER=$(realpath ./onedal).* echo $FILTER gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov -v --filter "${FITLER}" -o coverage"${1}".info sed -i "s|${PWD}/||g" coverage"${1}".info From 65c4cf24498d856e459c1d1f46b62bcee10974fd Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Fri, 10 Jan 2025 23:42:40 +0100 Subject: [PATCH 19/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 3fa3e115c7..13f20ff0da 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -29,7 +29,8 @@ if [[ $OSTYPE == *"linux"* ]]; then echo $GCOV_EXE FILTER=$(realpath ./onedal).* echo $FILTER - gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov -v --filter "${FITLER}" -o coverage"${1}".info - sed -i "s|${PWD}/||g" coverage"${1}".info + cd build + gcovr --gcov-executable "${GCOV_EXE}" -r ../ . --lcov -v --filter "${FITLER}" -o ../coverage"${1}".info + sed -i "s|${PWD}/||g" ../coverage"${1}".info # remove absolute filepath to match coverage.py file fi From 3878f8be7d2f91d0792c31947e62f8682c081d6d Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 07:31:52 +0100 Subject: [PATCH 20/72] Update ci.yml --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bf53466d6..9ad5a91647 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,18 +108,18 @@ jobs: bash .ci/scripts/describe_system.sh - name: Install develop requirements run: | - python -m venv venv + python -m venv ../venv source venv/bin/activate pip install -r dependencies-dev pip list - name: Build daal4py/sklearnex run: | - source venv/bin/activate + source ../venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} python setup.py install --single-version-externally-managed --record=record.txt - name: Install testing requirements run: | - source venv/bin/activate + source ../venv/bin/activate source .github/scripts/activate_components.sh bash .ci/scripts/setup_sklearn.sh ${{ matrix.SKLEARN_VERSION }} pip install --upgrade -r requirements-test.txt @@ -128,21 +128,21 @@ jobs: pip list - name: Sklearnex testing run: | - source venv/bin/activate + source ../venv/bin/activate source .github/scripts/activate_components.sh export COVERAGE_FILE=$(pwd)/.coverage.sklearnex cd .ci ../conda-recipe/run_test.sh - name: Sklearn testing run: | - source venv/bin/activate + source ../venv/bin/activate source .github/scripts/activate_components.sh export COVERAGE_FILE=$(pwd)/.coverage.sklearn if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi bash .ci/scripts/run_sklearn_tests.sh $CPU - name: Create coverage reports run: | - source venv/bin/activate + source ../venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }} - name: Archive coverage report @@ -154,7 +154,7 @@ jobs: coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json - name: Sklearn testing [preview] run: | - source venv/bin/activate + source ../venv/bin/activate source .github/scripts/activate_components.sh if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi export SKLEARNEX_PREVIEW='YES' From 84bd99d7e43404c70ed548ae417184ee13e0aeba Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 09:23:20 +0100 Subject: [PATCH 21/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ad5a91647..74854a675e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: - name: Install develop requirements run: | python -m venv ../venv - source venv/bin/activate + source ../venv/bin/activate pip install -r dependencies-dev pip list - name: Build daal4py/sklearnex From b301863267036216fa360aa6378d9be621099975 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 12:40:40 +0100 Subject: [PATCH 22/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74854a675e..9d908e3906 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,6 @@ jobs: run: | source ../venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} - python setup.py install --single-version-externally-managed --record=record.txt - name: Install testing requirements run: | source ../venv/bin/activate @@ -125,6 +124,7 @@ jobs: pip install --upgrade -r requirements-test.txt pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml gcovr if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then pip install dpctl==${{ env.DPCTL_VERSION }} dpnp==${{ env.DPNP_VERSION }}; fi + python setup.py install --single-version-externally-managed --record=record.txt pip list - name: Sklearnex testing run: | From df19a3fc12157bfbc1657974d8250fdf79c54859 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 13:34:13 +0100 Subject: [PATCH 23/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 13f20ff0da..378f764d5e 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -24,7 +24,7 @@ if [[ $OSTYPE == *"linux"* ]]; then if [[ -z "$2" ]]; then GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" else - GCOV_EXE="gcov" + GCOV_EXE="gcov-11" fi echo $GCOV_EXE FILTER=$(realpath ./onedal).* From de9d7012577e02e31509fb919fa96ed48f9d601f Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 13:35:13 +0100 Subject: [PATCH 24/72] Update ci.yml --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d908e3906..2259cc06df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,12 +145,14 @@ jobs: source ../venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }} + ls -lha + ls -lha .. - name: Archive coverage report uses: actions/upload-artifact@v4 with: name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} path: | - coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info + coveragelnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json - name: Sklearn testing [preview] run: | From c2a6f12ab67fec3667a1c5789c02c82feda40092 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 14:12:11 +0100 Subject: [PATCH 25/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 378f764d5e..c702520704 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -24,7 +24,8 @@ if [[ $OSTYPE == *"linux"* ]]; then if [[ -z "$2" ]]; then GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" else - GCOV_EXE="gcov-11" + GCOV_EXE="gcov-13" + gcov --version fi echo $GCOV_EXE FILTER=$(realpath ./onedal).* From 5b5caa003f4b6b16652feb3d5605ee4a1cfcfc07 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 15:04:31 +0100 Subject: [PATCH 26/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index c702520704..4c3f838cee 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -24,7 +24,8 @@ if [[ $OSTYPE == *"linux"* ]]; then if [[ -z "$2" ]]; then GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" else - GCOV_EXE="gcov-13" + GCOV_EXE="gcov-4" + g++ --version gcov --version fi echo $GCOV_EXE From 10c1ad8255f938a2996d403deae100697aff94bc Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 15:38:35 +0100 Subject: [PATCH 27/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2259cc06df..8ccf670b5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT" # enable coverage report generation echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV" - echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV" + if [[ -z $DPCFLAG ]]; then echo "SKLEARNEX_GCOV=1" >> "$GITHUB_ENV"; fi - name: apt-get run: sudo apt-get update && sudo apt-get install -y clang-format - name: dpcpp installation From 613a6bf981b279267b856c4a3efb30b97dd3c6b3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 19:25:52 +0100 Subject: [PATCH 28/72] Update run_test.bat --- conda-recipe/run_test.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat index 2be86075d2..59519b2801 100644 --- a/conda-recipe/run_test.bat +++ b/conda-recipe/run_test.bat @@ -34,7 +34,7 @@ if "%PYTHON%"=="python" ( set "PYTEST_ARGS= " -IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-report= %PYTEST_ARGS%") +IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-branch --cov-report= %PYTEST_ARGS%") rem Note: execute with argument --json-report as second argument rem in order to produce a JSON report under folder '.pytest_reports'. From c75446ac9dbb7de1511109ecc39a8ce05fd01975 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sat, 11 Jan 2025 19:27:03 +0100 Subject: [PATCH 29/72] Update run_test.sh --- conda-recipe/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index 92e8b97672..28620adfb5 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -55,7 +55,7 @@ function generate_pytest_args { ARGS+=("--json-report-file=.pytest_reports/$1_report.json") fi if [ -n "${COVERAGE_RCFILE}" ]; then - ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-report=) + ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-branch --cov-report=) fi printf -- "${ARGS[*]}" } From 036af4d3a14b112b250256bdecb36c1957d3408a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 07:30:27 +0100 Subject: [PATCH 30/72] Update ci.yml --- .github/workflows/ci.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ccf670b5e..82d686235e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,45 +108,44 @@ jobs: bash .ci/scripts/describe_system.sh - name: Install develop requirements run: | - python -m venv ../venv + python -m venv venv source ../venv/bin/activate pip install -r dependencies-dev pip list - name: Build daal4py/sklearnex run: | - source ../venv/bin/activate + source venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} + python setup.py install --single-version-externally-managed --record=record.txt + if [[ -n "${SKLEARNEX_GCOV}" ]]; then echo "NUMPY_GCOV=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV"; fi - name: Install testing requirements run: | - source ../venv/bin/activate + source venv/bin/activate source .github/scripts/activate_components.sh bash .ci/scripts/setup_sklearn.sh ${{ matrix.SKLEARN_VERSION }} pip install --upgrade -r requirements-test.txt pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml gcovr if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then pip install dpctl==${{ env.DPCTL_VERSION }} dpnp==${{ env.DPNP_VERSION }}; fi - python setup.py install --single-version-externally-managed --record=record.txt pip list - name: Sklearnex testing run: | - source ../venv/bin/activate + source venv/bin/activate source .github/scripts/activate_components.sh export COVERAGE_FILE=$(pwd)/.coverage.sklearnex cd .ci ../conda-recipe/run_test.sh - name: Sklearn testing run: | - source ../venv/bin/activate + source venv/bin/activate source .github/scripts/activate_components.sh export COVERAGE_FILE=$(pwd)/.coverage.sklearn if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi bash .ci/scripts/run_sklearn_tests.sh $CPU - name: Create coverage reports run: | - source ../venv/bin/activate + source venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} bash .github/scripts/generate_coverage_reports.sh lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }} - ls -lha - ls -lha .. - name: Archive coverage report uses: actions/upload-artifact@v4 with: @@ -156,7 +155,7 @@ jobs: coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json - name: Sklearn testing [preview] run: | - source ../venv/bin/activate + source venv/bin/activate source .github/scripts/activate_components.sh if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi export SKLEARNEX_PREVIEW='YES' From 12b04801906d127e262ff8a8028a65ac1b002238 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 07:34:47 +0100 Subject: [PATCH 31/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82d686235e..02791ee1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: - name: Install develop requirements run: | python -m venv venv - source ../venv/bin/activate + source venv/bin/activate pip install -r dependencies-dev pip list - name: Build daal4py/sklearnex From 45b597bfaf431c998d0b248860394a2218c071c9 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 07:57:42 +0100 Subject: [PATCH 32/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 4c3f838cee..8ec34abe3d 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -24,15 +24,22 @@ if [[ $OSTYPE == *"linux"* ]]; then if [[ -z "$2" ]]; then GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" else - GCOV_EXE="gcov-4" - g++ --version - gcov --version + GCOV_EXE="gcov" fi echo $GCOV_EXE FILTER=$(realpath ./onedal).* echo $FILTER - cd build - gcovr --gcov-executable "${GCOV_EXE}" -r ../ . --lcov -v --filter "${FITLER}" -o ../coverage"${1}".info - sed -i "s|${PWD}/||g" ../coverage"${1}".info + + NUMPY_TEST=$(python -m pip freeze | grep numpy) + # install dependencies + # proper operation of gcov with sklearnex requires the header files from + # the build numpy, this must be previously set as NUMPY_GCOV env. variable + python -m pip install gcovr $NUMPY_GCOV + + gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o ../coverage"${1}".info # remove absolute filepath to match coverage.py file + sed -i "s|${PWD}/||g" ../coverage"${1}".info + + # reinstall previous numpy + python -m pip install $NUMPY_TEST fi From a3f3324ba87a3edc5e277efe65fd64aa9716f88a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 07:58:24 +0100 Subject: [PATCH 33/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02791ee1ea..bed3fa306e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,7 +124,7 @@ jobs: source .github/scripts/activate_components.sh bash .ci/scripts/setup_sklearn.sh ${{ matrix.SKLEARN_VERSION }} pip install --upgrade -r requirements-test.txt - pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml gcovr + pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then pip install dpctl==${{ env.DPCTL_VERSION }} dpnp==${{ env.DPNP_VERSION }}; fi pip list - name: Sklearnex testing From 0cbf97ef13030ade5f576668bc64dc00428e677a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 08:50:55 +0100 Subject: [PATCH 34/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bed3fa306e..c3fb490522 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: source venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} python setup.py install --single-version-externally-managed --record=record.txt - if [[ -n "${SKLEARNEX_GCOV}" ]]; then echo "NUMPY_GCOV=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV"; fi + echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV" - name: Install testing requirements run: | source venv/bin/activate From d0dfa5c42423e372bc796d112d2244d44675c6eb Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 08:51:32 +0100 Subject: [PATCH 35/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 8ec34abe3d..0975e92da6 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -33,8 +33,8 @@ if [[ $OSTYPE == *"linux"* ]]; then NUMPY_TEST=$(python -m pip freeze | grep numpy) # install dependencies # proper operation of gcov with sklearnex requires the header files from - # the build numpy, this must be previously set as NUMPY_GCOV env. variable - python -m pip install gcovr $NUMPY_GCOV + # the build numpy, this must be previously set as NUMPY_BUILD + python -m pip install gcovr $NUMPY_BUILD gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o ../coverage"${1}".info # remove absolute filepath to match coverage.py file From 2b4e8c4192217507663321df0a3c6922129c8f2f Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 09:15:03 +0100 Subject: [PATCH 36/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3fb490522..98172ba52f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,12 +112,12 @@ jobs: source venv/bin/activate pip install -r dependencies-dev pip list + echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV" - name: Build daal4py/sklearnex run: | source venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} python setup.py install --single-version-externally-managed --record=record.txt - echo "NUMPY_BUILD=$(python -m pip freeze | grep numpy)" >> "$GITHUB_ENV" - name: Install testing requirements run: | source venv/bin/activate From 3a414678199ebe35782738939332bd9521ada845 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 09:15:23 +0100 Subject: [PATCH 37/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 0975e92da6..70e85f1523 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -36,7 +36,7 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r build/ --lcov --filter "${FITLER}" -o ../coverage"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . --lcov --filter "${FITLER}" -o ../coverage"${1}".info # remove absolute filepath to match coverage.py file sed -i "s|${PWD}/||g" ../coverage"${1}".info From 60d1ff892641191c119b5dd647abe95c477b96e7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 09:15:39 +0100 Subject: [PATCH 38/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 70e85f1523..656eb5ec0a 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -36,7 +36,7 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . --lcov --filter "${FITLER}" -o ../coverage"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o ../coverage"${1}".info # remove absolute filepath to match coverage.py file sed -i "s|${PWD}/||g" ../coverage"${1}".info From 79307eb5259b7334a3b9db8c275653d6d87c01b1 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 09:55:12 +0100 Subject: [PATCH 39/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 656eb5ec0a..543e7c3c0e 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -36,7 +36,7 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o ../coverage"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o coverage"${1}".info # remove absolute filepath to match coverage.py file sed -i "s|${PWD}/||g" ../coverage"${1}".info From fe7a927bbe71b36610e51ceb7b161067c08eebbb Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 10:45:43 +0100 Subject: [PATCH 40/72] Update run_sklearn_tests.py --- .ci/scripts/run_sklearn_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/scripts/run_sklearn_tests.py b/.ci/scripts/run_sklearn_tests.py index a7f5b04b7f..4dac925b2f 100644 --- a/.ci/scripts/run_sklearn_tests.py +++ b/.ci/scripts/run_sklearn_tests.py @@ -58,6 +58,7 @@ pytest_args += ( "--cov=onedal", "--cov=sklearnex", + "--cov-branch", f"--cov-config={rc}", "--cov-report=", ) From b02aac9944a925e0122edc40a00032404baa961e Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 13:31:06 +0100 Subject: [PATCH 41/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 543e7c3c0e..4ee51b8349 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -14,9 +14,12 @@ # limitations under the License. #=============================================================================== +ci_dir=$( dirname $( dirname "${BASH_SOURCE[0]}" ) ) +cd $ci_dir + # create coverage.py report coverage combine .coverage.sklearnex .coverage.sklearn -coverage json -o coverage."${1}".json +coverage lcov -o coverage_"${1}".info # create gcov report (lcov format) if [[ $OSTYPE == *"linux"* ]]; then @@ -36,9 +39,9 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o coverage"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o gcov_"${1}".info # remove absolute filepath to match coverage.py file - sed -i "s|${PWD}/||g" ../coverage"${1}".info + sed -i "s|${PWD}/||g" ../gcov_"${1}".info # reinstall previous numpy python -m pip install $NUMPY_TEST From 5e45731c0abfc6f7d93e4a12a98cec185cbf2344 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 13:32:43 +0100 Subject: [PATCH 42/72] Update ci.yml --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98172ba52f..1f8280e2be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,7 @@ jobs: export COVERAGE_FILE=$(pwd)/.coverage.sklearn if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi bash .ci/scripts/run_sklearn_tests.sh $CPU - - name: Create coverage reports + - name: Create coverage report run: | source venv/bin/activate source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }} @@ -151,8 +151,8 @@ jobs: with: name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} path: | - coveragelnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info - coverage.lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json + gcov_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info + coverage_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info - name: Sklearn testing [preview] run: | source venv/bin/activate @@ -297,7 +297,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: coverage_win_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} - path: coverage.win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json + path: coverage_win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json - name: Sklearn testing [preview] shell: cmd run: | From a70a2900a2acb082c1c62137a1faba860709b4d1 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 14:15:40 +0100 Subject: [PATCH 43/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 4ee51b8349..2a1107e6e8 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -14,12 +14,12 @@ # limitations under the License. #=============================================================================== -ci_dir=$( dirname $( dirname "${BASH_SOURCE[0]}" ) ) +ci_dir=$( dirname $( dirname $( dirname "${BASH_SOURCE[0]}" ) ) ) cd $ci_dir # create coverage.py report coverage combine .coverage.sklearnex .coverage.sklearn -coverage lcov -o coverage_"${1}".info +coverage lcov -o coverage."${1}".info # create gcov report (lcov format) if [[ $OSTYPE == *"linux"* ]]; then @@ -39,9 +39,7 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o gcov_"${1}".info - # remove absolute filepath to match coverage.py file - sed -i "s|${PWD}/||g" ../gcov_"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o gcov."${1}".info # reinstall previous numpy python -m pip install $NUMPY_TEST From 3e2c671705ba5cea02e20ec581d8ab84358283da Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 14:16:17 +0100 Subject: [PATCH 44/72] Update build_backend.py --- scripts/build_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_backend.py b/scripts/build_backend.py index a1fb716de7..1480890eab 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -133,8 +133,8 @@ def custom_build_cmake_clib( cmake_args += ["-DADD_ONEDAL_RPATH=ON"] if use_gcov: - cmake_args += ["-DONEDAL_GCOV=ON"] - + cmake_args += ["-DSKLEARNEX_GCOV=ON"] + cpu_count = multiprocessing.cpu_count() # limit parallel cmake jobs if memory size is insufficient # TODO: add on all platforms From 6d7d84175b8b2f6f87564eff1bc73d4dc4e398a7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 14:17:13 +0100 Subject: [PATCH 45/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index b39f948005..6a00de888a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -29,8 +29,8 @@ set(CMAKE_CXX_EXTENSIONS OFF) option(ADD_ONEDAL_RPATH "Adds oneDAL's file paths to the RPATH here" OFF) message(STATUS "ADD_ONEDAL_RPATH:" ${ADD_ONEDAL_RPATH}) -option(ONEDAL_GCOV "Compile with gcov" OFF) -message(STATUS "ONEDAL_GCOV:" ${ONEDAL_GCOV}) +option(SKLEARNEX_GCOV "Compile with gcov" OFF) +message(STATUS "SKLEARNEX_GCOV:" ${ONEDAL_SKLEARNEX}) if(WIN32) # hint CMake to get python from PYTHON env. variable if defined @@ -57,7 +57,7 @@ else() message(FATAL_ERROR "Unsupported system.") endif() -if(ONEDAL_GCOV) +if(SKLEARNEX_GCOV) if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") @@ -149,7 +149,7 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) endif() - if(ONEDAL_GCOV) + if(SKLEARNEX_GCOV) #assumes first argument is --coverage, allowing host code profiling set(CMAKE_CXX_FLAGS "-Xarch_host ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host ${CMAKE_SHARED_LINKER_FLAGS}") From 462196bbad3611e2b850a494201a2f2c822916fe Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 14:18:09 +0100 Subject: [PATCH 46/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 2a1107e6e8..05d2420d8b 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -19,7 +19,7 @@ cd $ci_dir # create coverage.py report coverage combine .coverage.sklearnex .coverage.sklearn -coverage lcov -o coverage."${1}".info +coverage lcov -o coverage_"${1}".info # create gcov report (lcov format) if [[ $OSTYPE == *"linux"* ]]; then @@ -39,7 +39,7 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o gcov."${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o gcov_"${1}".info # reinstall previous numpy python -m pip install $NUMPY_TEST From 806ec607c6b553e2b0ff3dab4495e9b0edd912a9 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 14:26:56 +0100 Subject: [PATCH 47/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 6a00de888a..5963070f9b 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -30,7 +30,7 @@ option(ADD_ONEDAL_RPATH "Adds oneDAL's file paths to the RPATH here" OFF) message(STATUS "ADD_ONEDAL_RPATH:" ${ADD_ONEDAL_RPATH}) option(SKLEARNEX_GCOV "Compile with gcov" OFF) -message(STATUS "SKLEARNEX_GCOV:" ${ONEDAL_SKLEARNEX}) +message(STATUS "SKLEARNEX_GCOV:" ${SKLEARNEX_GCOV}) if(WIN32) # hint CMake to get python from PYTHON env. variable if defined From b4a9a3337be54e61d219184e59d7359c0374d922 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 14:30:14 +0100 Subject: [PATCH 48/72] Update ci.yml --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f8280e2be..3d7c3ad9c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,8 +151,7 @@ jobs: with: name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} path: | - gcov_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info - coverage_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info + *_lnx${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info - name: Sklearn testing [preview] run: | source venv/bin/activate From 945e802e6fe007bbfcdc416ccbcbf3cc3a3f58d3 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 15:16:15 +0100 Subject: [PATCH 49/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 05d2420d8b..b166fa24af 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -19,7 +19,7 @@ cd $ci_dir # create coverage.py report coverage combine .coverage.sklearnex .coverage.sklearn -coverage lcov -o coverage_"${1}".info +coverage lcov -o coverage_py_"${1}".info # create gcov report (lcov format) if [[ $OSTYPE == *"linux"* ]]; then @@ -39,7 +39,7 @@ if [[ $OSTYPE == *"linux"* ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o gcov_"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o coverage_cpp_"${1}".info # reinstall previous numpy python -m pip install $NUMPY_TEST From 7924ef912ea875751e205864d1e447534f0f614b Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 15:17:56 +0100 Subject: [PATCH 50/72] Update ci.yml --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d7c3ad9c9..69173606d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,7 +296,8 @@ jobs: uses: actions/upload-artifact@v4 with: name: coverage_win_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} - path: coverage_win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.json + path: | + *_win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}.info - name: Sklearn testing [preview] shell: cmd run: | From 3374015b364d0d8c3314047cc39df01a3967bdc6 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 15:18:36 +0100 Subject: [PATCH 51/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 5963070f9b..7a6c58ad0a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -161,7 +161,7 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_C_FLAGS "-fsycl ${CMAKE_C_FLAGS}") - endif() + endif() set(ONEDAL_LIBRARIES "") if(WIN32) From 82608c189bdccf7e0d2a290f1ef6546c1aebe6e1 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 21:48:25 +0100 Subject: [PATCH 52/72] Update ci.yml --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69173606d3..6d3a2c43c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,6 +245,7 @@ jobs: pip install --upgrade setuptools pip install cpufeature clang-format pyyaml pip install -r dependencies-dev + for /f "delims=" %%c in ('python -m pip freeze | grep numpy') do echo NUMPY_BUILD=%%c>> %GITHUB_ENV% - name: System info shell: cmd run: | From 1bf7677dcc34fab31e4f23acc1a6464bedb05118 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 21:58:57 +0100 Subject: [PATCH 53/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d3a2c43c0..2fd438e626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,7 +245,7 @@ jobs: pip install --upgrade setuptools pip install cpufeature clang-format pyyaml pip install -r dependencies-dev - for /f "delims=" %%c in ('python -m pip freeze | grep numpy') do echo NUMPY_BUILD=%%c>> %GITHUB_ENV% + for /f "delims=" %%c in ('python -m pip freeze ^| grep numpy') do echo NUMPY_BUILD=%%c>> %GITHUB_ENV% - name: System info shell: cmd run: | From 52b02005d60a9e28064d626807c08f39f90cf039 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 22:08:24 +0100 Subject: [PATCH 54/72] Update ci.yml --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fd438e626..417b3b777f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,6 +223,7 @@ jobs: set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}" if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%) echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV% + if "${{ steps.set-env.outputs.DPCFLAG }}"=="" echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV% - name: Download Intel OpenCL CPU Runtime artifact if: ${{ steps.set-env.outputs.DPCFLAG == '' }} uses: actions/download-artifact@v4 @@ -292,6 +293,7 @@ jobs: shell: cmd run: | call .\venv\Scripts\activate.bat + call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }} bash .github/scripts/generate_coverage_reports.sh win${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }} - name: Archive coverage report uses: actions/upload-artifact@v4 From d0b8a74234bc760cf578133f3a2ca1f012e34bdf Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 22:12:20 +0100 Subject: [PATCH 55/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index b166fa24af..60375a8b01 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -22,7 +22,7 @@ coverage combine .coverage.sklearnex .coverage.sklearn coverage lcov -o coverage_py_"${1}".info # create gcov report (lcov format) -if [[ $OSTYPE == *"linux"* ]]; then +if [[ -n "${SKLEARNEX_GCOV}" ]]; then # extract llvm tool for gcov processing if [[ -z "$2" ]]; then GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" From 436e306d32f1dcfd8dcc04e8aa4c515b2ab400ca Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 22:13:00 +0100 Subject: [PATCH 56/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index 60375a8b01..f93fb93a6e 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -25,7 +25,7 @@ coverage lcov -o coverage_py_"${1}".info if [[ -n "${SKLEARNEX_GCOV}" ]]; then # extract llvm tool for gcov processing if [[ -z "$2" ]]; then - GCOV_EXE="$(dirname $(type -P -a icpx))/compiler/llvm-cov gcov" + GCOV_EXE="$(dirname $(type -P -a icx))/compiler/llvm-cov gcov" else GCOV_EXE="gcov" fi From 9cba3b13c07a9b7c672bf3650909e778f77edd3a Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 22:56:42 +0100 Subject: [PATCH 57/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 7a6c58ad0a..891fd13515 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -57,15 +57,6 @@ else() message(FATAL_ERROR "Unsupported system.") endif() -if(SKLEARNEX_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") - else() - message(FATAL_ERROR "Unsupported compiler for code coverage.") - endif() -endif() - set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS} ${SDL_FLAGS}") @@ -110,6 +101,15 @@ find_package(pybind11 REQUIRED) if(IFACE STREQUAL "host") set(TARGET "_onedal_py_host") + if(SKLEARNEX_GCOV) + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(WARNING "Code coverage will not be generated for `host` target") + endif() + endif() + set(ONEDAL_LIBRARIES "") if(WIN32) list(APPEND ONEDAL_LIBRARIES "onedal_dll.${ONEDAL_MAJOR_BINARY}") @@ -150,13 +150,17 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) endif() if(SKLEARNEX_GCOV) - #assumes first argument is --coverage, allowing host code profiling - set(CMAKE_CXX_FLAGS "-Xarch_host ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host ${CMAKE_SHARED_LINKER_FLAGS}") + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(WARNING "Code coverage will not be generated for `${IFACE}` target") + endif() endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_CXX_FLAGS "-fsycl ${CMAKE_CXX_FLAGS}") + endif() if(CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") From d20dc6f7f2b3f2a053b4b22b581aafadde47a9b6 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 23:26:34 +0100 Subject: [PATCH 58/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 891fd13515..de7956bbf2 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -150,10 +150,13 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) endif() if(SKLEARNEX_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32) + set(CMAKE_CXX_FLAGS "/Xarch_host --coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "/Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND UNIX) set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") - else() + else message(WARNING "Code coverage will not be generated for `${IFACE}` target") endif() endif() From 9bc5eb424c446ead56349676725448a2248b0e0c Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 23:38:38 +0100 Subject: [PATCH 59/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index de7956bbf2..ccbb6dd76f 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -106,7 +106,7 @@ if(IFACE STREQUAL "host") set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") else() - message(WARNING "Code coverage will not be generated for `host` target") + message(WARNING "Code coverage will not be generated for target: host") endif() endif() @@ -157,7 +157,7 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") else - message(WARNING "Code coverage will not be generated for `${IFACE}` target") + message(WARNING "Code coverage will not be generated for target: " ${IFACE}) endif() endif() From ac5c25afbb662269635c763fd976c336742194b1 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Sun, 12 Jan 2025 23:45:48 +0100 Subject: [PATCH 60/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index ccbb6dd76f..a77dee8b5a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -156,7 +156,7 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND UNIX) set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") - else + else() message(WARNING "Code coverage will not be generated for target: " ${IFACE}) endif() endif() From d725fbf95de6f6b4eeb72595cac13dd3d4d38f61 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 00:03:24 +0100 Subject: [PATCH 61/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index a77dee8b5a..84f72bb4f6 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -151,8 +151,8 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) if(SKLEARNEX_GCOV) if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32) - set(CMAKE_CXX_FLAGS "/Xarch_host --coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "/Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_CXX_FLAGS "/clang:-Xarch_host /clang:--coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "/clang:-Xarch_host /clang:--coverage ${CMAKE_SHARED_LINKER_FLAGS}") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND UNIX) set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") From 58774c480f2abf64ce232cdb3878d392fa26f571 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 01:05:57 +0100 Subject: [PATCH 62/72] Update build_backend.py --- scripts/build_backend.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 1480890eab..e453af9c66 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -134,6 +134,11 @@ def custom_build_cmake_clib( if use_gcov: cmake_args += ["-DSKLEARNEX_GCOV=ON"] + if IS_WIN and cxx == "icx": + # Windows builds using the MSVC linker, requiring manually specifying + # the clang_rt.profile-x86_64.lib and path, which must be determined + # from the icx compiler. It is passed as an additional input. + pass cpu_count = multiprocessing.cpu_count() # limit parallel cmake jobs if memory size is insufficient From dbf8114d38f96ffd737286cfa9ec2565c2ec70ff Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 01:22:19 +0100 Subject: [PATCH 63/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 84f72bb4f6..6d69919e12 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -101,15 +101,6 @@ find_package(pybind11 REQUIRED) if(IFACE STREQUAL "host") set(TARGET "_onedal_py_host") - if(SKLEARNEX_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") - else() - message(WARNING "Code coverage will not be generated for target: host") - endif() - endif() - set(ONEDAL_LIBRARIES "") if(WIN32) list(APPEND ONEDAL_LIBRARIES "onedal_dll.${ONEDAL_MAJOR_BINARY}") @@ -134,6 +125,15 @@ if(IFACE STREQUAL "host") endif() endif() + if(SKLEARNEX_GCOV) + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(WARNING "Code coverage will not be generated for target: host") + endif() + endif() + list(APPEND COMPILE_DEFINITIONS "NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION") elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) @@ -149,18 +149,6 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) endif() - if(SKLEARNEX_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32) - set(CMAKE_CXX_FLAGS "/clang:-Xarch_host /clang:--coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "/clang:-Xarch_host /clang:--coverage ${CMAKE_SHARED_LINKER_FLAGS}") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND UNIX) - set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") - else() - message(WARNING "Code coverage will not be generated for target: " ${IFACE}) - endif() - endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") set(CMAKE_CXX_FLAGS "-fsycl ${CMAKE_CXX_FLAGS}") @@ -194,6 +182,18 @@ elseif(IFACE_IS_DPC OR IFACE_IS_SPMD_DPC) endif() endif() + if(SKLEARNEX_GCOV) + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32) + set(CMAKE_CXX_FLAGS "/clang:-Xarch_host /clang:--coverage ${CMAKE_CXX_FLAGS}") + list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND UNIX) + set(CMAKE_CXX_FLAGS "-Xarch_host --coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "-Xarch_host --coverage ${CMAKE_SHARED_LINKER_FLAGS}") + else() + message(WARNING "Code coverage will not be generated for target: " ${IFACE}) + endif() + endif() + if(IFACE_IS_SPMD_DPC) set(MPI_LIBRARY ${MPI_LIBS}) endif() From 862141bfe80c4ca3e7642d68bbd8546a9b0d2a5e Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 02:16:53 +0100 Subject: [PATCH 64/72] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 417b3b777f..fb6323ea20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,7 +223,7 @@ jobs: set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}" if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%) echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV% - if "${{ steps.set-env.outputs.DPCFLAG }}"=="" echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV% + if %DPCFLAG%=="" echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV% - name: Download Intel OpenCL CPU Runtime artifact if: ${{ steps.set-env.outputs.DPCFLAG == '' }} uses: actions/download-artifact@v4 From b772d97f798dbf2b404f81cc68f13a31418531c0 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 02:18:21 +0100 Subject: [PATCH 65/72] Update generate_coverage_reports.sh --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index f93fb93a6e..e97e402243 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -14,7 +14,7 @@ # limitations under the License. #=============================================================================== -ci_dir=$( dirname $( dirname $( dirname "${BASH_SOURCE[0]}" ) ) ) +ci_dir=$(dirname $(dirname $(dirname "${BASH_SOURCE[0]}"))) cd $ci_dir # create coverage.py report From a737345501a07333c3852a6057d1c90b054a62a7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 02:19:06 +0100 Subject: [PATCH 66/72] Update build_backend.py --- scripts/build_backend.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/build_backend.py b/scripts/build_backend.py index e453af9c66..a6949f8145 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -119,7 +119,6 @@ def custom_build_cmake_clib( "-DoneDAL_LIBRARY_DIR=" + jp(os.environ["DALROOT"], "lib", arch_dir), "-Dpybind11_DIR=" + pybind11.get_cmake_dir(), "-DoneDAL_USE_PARAMETERS_LIB=" + use_parameters_arg, - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", ] if build_distribute: @@ -134,11 +133,6 @@ def custom_build_cmake_clib( if use_gcov: cmake_args += ["-DSKLEARNEX_GCOV=ON"] - if IS_WIN and cxx == "icx": - # Windows builds using the MSVC linker, requiring manually specifying - # the clang_rt.profile-x86_64.lib and path, which must be determined - # from the icx compiler. It is passed as an additional input. - pass cpu_count = multiprocessing.cpu_count() # limit parallel cmake jobs if memory size is insufficient From 728b83ec38b7433168604ebf1a3fa1da4e3f9984 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 02:21:21 +0100 Subject: [PATCH 67/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 6d69919e12..baf188acd8 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -126,7 +126,10 @@ if(IFACE STREQUAL "host") endif() if(SKLEARNEX_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32) + set(CMAKE_CXX_FLAGS "/clang:--coverage ${CMAKE_CXX_FLAGS}") + list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") else() From d1e55b1433e31f0896a5eb8d1179e547e8ab294f Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 02:37:49 +0100 Subject: [PATCH 68/72] Update ci.yml --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb6323ea20..248f7cb0fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -221,9 +221,13 @@ jobs: echo C:\msys64\usr\bin;>> %GITHUB_PATH% echo NO_DIST=YES>> %GITHUB_ENV% set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}" - if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%) + if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% ( + echo DPCFLAG=>> %GITHUB_OUTPUT% + echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV% + ) else ( + echo DPCFLAG="0">> %GITHUB_OUTPUT% + ) echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV% - if %DPCFLAG%=="" echo SKLEARNEX_GCOV=YES>> %GITHUB_ENV% - name: Download Intel OpenCL CPU Runtime artifact if: ${{ steps.set-env.outputs.DPCFLAG == '' }} uses: actions/download-artifact@v4 From 0290055868bcb196e16386fe00b97fc0d2d85438 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 10:08:49 +0100 Subject: [PATCH 69/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index baf188acd8..9b9579561c 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -126,10 +126,15 @@ if(IFACE STREQUAL "host") endif() if(SKLEARNEX_GCOV) - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND WIN32) - set(CMAKE_CXX_FLAGS "/clang:--coverage ${CMAKE_CXX_FLAGS}") - list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if((CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")) + if(WIN32) + set(CMAKE_CXX_FLAGS "/clang:--coverage ${CMAKE_CXX_FLAGS}") + list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib") + else() + set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "--coverage ${CMAKE_CXX_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "--coverage ${CMAKE_SHARED_LINKER_FLAGS}") else() From e5ecf2a0e5f894644e8dbeba0c2cba5453a6f6b7 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 10:09:12 +0100 Subject: [PATCH 70/72] Update CMakeLists.txt --- scripts/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 9b9579561c..236188df6d 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -126,7 +126,7 @@ if(IFACE STREQUAL "host") endif() if(SKLEARNEX_GCOV) - if((CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")) + if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") if(WIN32) set(CMAKE_CXX_FLAGS "/clang:--coverage ${CMAKE_CXX_FLAGS}") list(APPEND ONEDAL_LIBRARIES "clang_rt.profile-x86_64.lib") From 9b73ea6cdd3bb9005415ab38e05084d0e5196fe9 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Mon, 13 Jan 2025 13:35:00 +0100 Subject: [PATCH 71/72] switch azure to lcov --- .ci/pipeline/codecov-lnx.yml | 4 ++-- .ci/pipeline/codecov-win.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/pipeline/codecov-lnx.yml b/.ci/pipeline/codecov-lnx.yml index 102e7ff147..7ef026e3e8 100644 --- a/.ci/pipeline/codecov-lnx.yml +++ b/.ci/pipeline/codecov-lnx.yml @@ -18,13 +18,13 @@ steps: . /usr/share/miniconda/etc/profile.d/conda.sh conda activate CB coverage combine .coverage.sklearnex .coverage.sklearn - coverage json -o coverage.json + coverage lcov -o coverage.info displayName: "Create coverage report" - script: | curl -Os https://cli.codecov.io/latest/linux/codecov chmod +x codecov export VARARGS="-n azure-${AGENT_OS}-$(PYTHON_VERSION)-$(SKLEARN_VERSION)" - ./codecov -v upload-process -Z -t ${CODECOV_TOKEN} "${VARARGS}" -F azure -f coverage.json + ./codecov -v upload-process -Z -t ${CODECOV_TOKEN} "${VARARGS}" -F azure -f coverage.info displayName: "Upload to codecov" env: CODECOV_TOKEN: $(CODECOV_TOKEN) diff --git a/.ci/pipeline/codecov-win.yml b/.ci/pipeline/codecov-win.yml index 76bd60494b..e5cf4a62ce 100644 --- a/.ci/pipeline/codecov-win.yml +++ b/.ci/pipeline/codecov-win.yml @@ -17,13 +17,13 @@ steps: - script: | call activate CB coverage combine .coverage.sklearnex .coverage.sklearn - coverage json -o coverage.json + coverage lcov -o coverage.info displayName: "Create coverage report" - script: | set PATH=C:\msys64\usr\bin;%PATH% curl -Os https://cli.codecov.io/latest/windows/codecov.exe set VARARGS=-n azure-%AGENT_OS%-$(PYTHON_VERSION)-$(SKLEARN_VERSION) - .\codecov.exe -v upload-process -Z -t %CODECOV_TOKEN% %VARARGS% -F azure -f coverage.json + .\codecov.exe -v upload-process -Z -t %CODECOV_TOKEN% %VARARGS% -F azure -f coverage.info displayName: "Upload to codecov" env: CODECOV_TOKEN: $(CODECOV_TOKEN) From d085abe74cc06cd07087e8b5810c0ede0c2d7f8e Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 13 Jan 2025 14:21:17 +0100 Subject: [PATCH 72/72] Update .github/scripts/generate_coverage_reports.sh Co-authored-by: david-cortes-intel --- .github/scripts/generate_coverage_reports.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/generate_coverage_reports.sh b/.github/scripts/generate_coverage_reports.sh index e97e402243..72f2f9eab8 100644 --- a/.github/scripts/generate_coverage_reports.sh +++ b/.github/scripts/generate_coverage_reports.sh @@ -39,7 +39,7 @@ if [[ -n "${SKLEARNEX_GCOV}" ]]; then # the build numpy, this must be previously set as NUMPY_BUILD python -m pip install gcovr $NUMPY_BUILD - gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FITLER}" -o coverage_cpp_"${1}".info + gcovr --gcov-executable "${GCOV_EXE}" -r . -v --lcov --filter "${FILTER}" -o coverage_cpp_"${1}".info # reinstall previous numpy python -m pip install $NUMPY_TEST