diff --git a/CMakeLists.txt b/CMakeLists.txt index 363e219..4052146 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,45 @@ -# MakeLists files in this project can -# refer to the root source directory of the project as ${HELLO_SOURCE_DIR} and -# to the root binary directory of the project as ${HELLO_BINARY_DIR}. +# For example the framework can run without GEANT4, but ROOT is +# mandatory + + +# ************************************************************************** +# * Copyright(c) 1998-2014, ALICE Experiment at CERN, All rights reserved. * +# * * +# * Author: The ALICE Off-line Project. * +# * Contributors are mentioned in the code where appropriate. * +# * * +# * Permission to use, copy, modify and distribute this software and its * +# * documentation strictly for non-commercial purposes is hereby granted * +# * without fee, provided that the above copyright notice appears in all * +# * copies and that both the copyright notice and this permission notice * +# * appear in the supporting documentation. The authors make no claims * +# * about the suitability of this software for any purpose. It is * +# * provided "as is" without express or implied warranty. * +# ************************************************************************** + +# Module cmake_minimum_required (VERSION 2.8.11) project (3DPoissonSolverGPU) -# Recurse into the "Hello" and "Demo" subdirectories. This does not actually -# cause another cmake executable to run. The same process will walk through -# the project's entire directory structure. +find_package(CUDA) +if(NOT CUDA_FOUND) + message( FATAL_ERROR "NVIDIA CUDA package not found. Please install or set SC_CUDA=OFF" ) +else() + find_library(LIBCUDA_SO_PATH libcuda.so) + string(FIND ${LIBCUDA_SO_PATH} "-NOTFOUND" LIBCUDA_SO_PATH_NOTFOUND ) +endif(NOT CUDA_FOUND) +message( STATUS "Building SpaceCharge distortion framework with CUDA support" ) + +if(LIBCUDA_SO_PATH_NOTFOUND GREATER -1) + message( FATAL_ERROR "NVIDIA CUDA libcuda.so not found. Please install or set SC_CUDA=OFF" ) +endif(LIBCUDA_SO_PATH_NOTFOUND GREATER -1) + +#set nvcc flags +set(CUDA_NVCC_FLAGS -Wno-deprecated-gpu-targets --use_fast_math --maxrregcount 64 -O4 -Xcompiler -fPIC -Xptxas -O4 -gencode arch=compute_30,code=sm_30 -gencode arch=compute_35,code=sm_35 -gencode arch=compute_52,code=sm_52 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_60,code=sm_60) + +#for convenience + -set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake/cuda" ${CMAKE_MODULE_PATH}) -find_package(CUDA QUIET REQUIRED) find_package(Doxygen) if (DOXYGEN_FOUND) @@ -33,9 +63,55 @@ endif (DOXYGEN_FOUND) -set(CUDA_ATTACH_VS_BUILD_RULE_TO_CUDA_FILE OFF) -set(BUILD_SHARED_LIBS ON) -list(APPEND CUDA_NVCC_FLAGS) -add_subdirectory (gpulib) +# Module include folder +include_directories(. + interface + kernel + ) + + +#compile CUDA object file +cuda_compile(SCGPU_O kernel/PoissonSolver3DGPU.cu) + + + +#set it back +if(STDCXX11FOUND GREATER -1) + string ( REPLACE "-std=c++98" "-std=c++11" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" ) +endif() +#end of CUDA specific stuff; from here on build usual library + + +# Sources in alphabetical order +set(SRCS + interface/PoissonSolver3DCylindricalGPU.cxx + ${SCGPU_O} + ) + +# Headers from sources +set(CINTHDRS + interface/PoissonSolver3DCylindricalGPU.h + ) + +set(HDRS + ${CINTHDRS} + kernel/PoissonSolver3DGPU.h + ) + +enable_testing() +set(TARGET_NAME poissonsolvergpu) + + +add_library(${TARGET_NAME} SHARED ${SRCS}) +target_link_libraries(${TARGET_NAME} ${CUDA_LIBRARIES} ${LIBCUDA_SO_PATH}) + +set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_FLAGS "") + +# Installation +install(TARGETS ${TARGET_NAME} + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib) + +install(FILES ${HDRS} DESTINATION include) diff --git a/CMakeLists.txt.temp b/CMakeLists.txt.temp new file mode 100644 index 0000000..f0856f6 --- /dev/null +++ b/CMakeLists.txt.temp @@ -0,0 +1,44 @@ +# MakeLists files in this project can +# refer to the root source directory of the project as ${HELLO_SOURCE_DIR} and +# to the root binary directory of the project as ${HELLO_BINARY_DIR}. +cmake_minimum_required (VERSION 2.8.11) +project (3DPoissonSolverGPU) + +# Recurse into the "Hello" and "Demo" subdirectories. This does not actually +# cause another cmake executable to run. The same process will walk through +# the project's entire directory structure. + +set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake/cuda" ${CMAKE_MODULE_PATH}) +find_package(CUDA QUIET REQUIRED) +find_package(Doxygen) + +if (DOXYGEN_FOUND) + # set input and output files + set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in) + set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) + + # request to configure the file + configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) + message("Doxygen build started") + + # note the option ALL which allows to build the docs together with the application + add_custom_target( doc_doxygen ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" + VERBATIM ) +else (DOXYGEN_FOUND) + message("Doxygen need to be installed to generate the doxygen documentation") +endif (DOXYGEN_FOUND) + + +enable_testing() +set(TARGET_NAME poissonsolvergpu) + +set(CUDA_ATTACH_VS_BUILD_RULE_TO_CUDA_FILE OFF) +set(BUILD_SHARED_LIBS ON) +list(APPEND CUDA_NVCC_FLAGS) + +add_subdirectory (kernel) +add_subdirectory (interface) + diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 41740c9..a1c5f42 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -20,7 +20,9 @@ endif(LIBCUDA_SO_PATH_NOTFOUND GREATER -1) #set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake/cuda" ${CMAKE_MODULE_PATH}) #find_package(CUDA QUIET REQUIRED) -find_library(PSLIB libPoissonSolver3DGPU.so) +set(PSLIBNAME libPoissonSolver3DGPU.so) + +find_library(PSLIB ${PSLIBNAME}) string(FIND ${PSLIB} "-NOTFOUND" PSLIB_NOTFOUND ) if(PSLIB_NOTFOUND GREATER -1) diff --git a/gpulib/CMakeLists.txt b/interface/CMakeLists.txt similarity index 100% rename from gpulib/CMakeLists.txt rename to interface/CMakeLists.txt diff --git a/interface/PoissonSolver3DCylindricalGPU.cxx b/interface/PoissonSolver3DCylindricalGPU.cxx new file mode 100644 index 0000000..ae91d1e --- /dev/null +++ b/interface/PoissonSolver3DCylindricalGPU.cxx @@ -0,0 +1,114 @@ +/// \author Rifki Sadikin , Indonesian Institute of Sciences +/// \date Nov 20, 2017 + +#include +#include "PoissonSolver3DCylindricalGPU.h" + +/// +PoissonSolver3DCylindricalGPU::PoissonSolver3DCylindricalGPU() { + + fErrorConvF = new float [fMgParameters.nMGCycle]; + fErrorExactF = new float [fMgParameters.nMGCycle]; + +} + + +PoissonSolver3DCylindricalGPU::PoissonSolver3DCylindricalGPU(int nRRow, int nZColumn, int nPhiSlice) { + fNRRow = nRRow; + fNZColumn = nZColumn; + fPhiSlice = nPhiSlice; + fErrorConvF = new float [fMgParameters.nMGCycle]; + fErrorExactF = new float [fMgParameters.nMGCycle]; +} +/// destructor +PoissonSolver3DCylindricalGPU::~PoissonSolver3DCylindricalGPU() { + delete fErrorConvF; + delete fErrorExactF; + delete fExactSolutionF; +} + +/// function overriding +void PoissonSolver3DCylindricalGPU::PoissonSolver3D(float *matricesV, float *matricesCharge, + int nRRow, int nZColumn, int phiSlice, int maxIteration, + int symmetry) { + + fNRRow = nRRow; + fNZColumn = nZColumn; + fPhiSlice = phiSlice; + + PoissonMultiGrid3D2D(matricesV, matricesCharge, nRRow, nZColumn, phiSlice, symmetry); + + +} + + +// method to do multigrid3d2d +void PoissonSolver3DCylindricalGPU::PoissonMultiGrid3D2D(float *VPotential, float * RhoChargeDensities, int nRRow, + int nZColumn, int phiSlice, int symmetry) { + + const float gridSizeR = (PoissonSolver3DCylindricalGPU::fgkOFCRadius-PoissonSolver3DCylindricalGPU::fgkIFCRadius) / (nRRow-1); // h_{r} + const float gridSizePhi = M_PI/phiSlice; // h_{phi} + const float gridSizeZ = PoissonSolver3DCylindricalGPU::fgkTPCZ0 / (nZColumn-1) ; // h_{z} + const float ratioPhi = gridSizeR*gridSizeR / (gridSizePhi*gridSizePhi) ; // ratio_{phi} = gridsize_{r} / gridsize_{phi} + const float ratioZ = gridSizeR*gridSizeR / (gridSizeZ*gridSizeZ) ; // ratio_{Z} = gridsize_{r} / gridsize_{z} + const float convErr = PoissonSolver3DCylindricalGPU::fgConvergenceError; + const float IFCRadius = PoissonSolver3DCylindricalGPU::fgkIFCRadius; + + int fparamsize = 8; + float * fparam = new float[fparamsize]; + + fparam[0] = gridSizeR; + fparam[1] = gridSizePhi; + fparam[2] = gridSizeZ; + fparam[3] = ratioPhi; + fparam[4] = ratioZ; + fparam[5] = convErr; + fparam[6] = IFCRadius; + + int iparamsize = 4; + int * iparam = new int[iparamsize]; + + iparam[0] = fMgParameters.nPre; + iparam[1] = fMgParameters.nPost; + iparam[2] = fMgParameters.maxLoop; + iparam[3] = fMgParameters.nMGCycle; + + + if (fMgParameters.cycleType == kFCycle) + { + if (fExactPresent == true) { + PoissonMultigrid3DSemiCoarseningGPUErrorFCycle(VPotential, RhoChargeDensities,nRRow, nZColumn,phiSlice,symmetry, fparam, iparam, fExactPresent, fErrorConvF, fErrorExactF, fExactSolutionF); + } else { + PoissonMultigrid3DSemiCoarseningGPUErrorFCycle(VPotential, RhoChargeDensities,nRRow, nZColumn,phiSlice,symmetry, fparam, iparam, fExactPresent, fErrorConvF, fErrorExactF, NULL); + + } + } else if (fMgParameters.cycleType == kWCycle) + { + PoissonMultigrid3DSemiCoarseningGPUErrorWCycle(VPotential, RhoChargeDensities,nRRow, nZColumn,phiSlice,symmetry, fparam, iparam, fErrorConvF, fErrorExactF, fExactSolutionF); + } else + { + if (fExactPresent == true) { + PoissonMultigrid3DSemiCoarseningGPUError(VPotential, RhoChargeDensities,nRRow, nZColumn,phiSlice,symmetry, fparam, iparam, fExactPresent, fErrorConvF, fErrorExactF, fExactSolutionF); + } else { + PoissonMultigrid3DSemiCoarseningGPUError(VPotential, RhoChargeDensities,nRRow, nZColumn,phiSlice,symmetry, fparam, iparam, fExactPresent, fErrorConvF, fErrorExactF, NULL); + } + } + fIterations = iparam[3]; + delete[] fparam; + delete[] iparam; +} + + + +void PoissonSolver3DCylindricalGPU::SetExactSolution(float*exactSolution,int nRRow, int nZColumn, int phiSlice) { + fNRRow = nRRow; + fNZColumn = nZColumn; + fPhiSlice = phiSlice; + fExactSolutionF = new float[fNRRow * fPhiSlice,fNZColumn]; + fExactPresent = true; + fMaxExact = 0.0;; + for (int i=0;i fMaxExact) fMaxExact = abs(fExactSolutionF[i]); + } +} diff --git a/interface/PoissonSolver3DCylindricalGPU.h b/interface/PoissonSolver3DCylindricalGPU.h new file mode 100644 index 0000000..feaf096 --- /dev/null +++ b/interface/PoissonSolver3DCylindricalGPU.h @@ -0,0 +1,129 @@ +#ifndef POISSONSOLVER3DCYLINDRICALGPU_H +#define POISSONSOLVER3DCYLINDRICALGPU_H + +/// \author Rifki Sadikin , Indonesian Institute of Sciences +/// \date Nov 20, 2017 +#include "PoissonSolver3DGPU.h" + + + +class PoissonSolver3DCylindricalGPU { +public: + ///< Enumeration of Poisson Solver Strategy Type + enum StrategyType { + kRelaxation = 0, ///< S.O.R Cascaded MultiGrid + kMultiGrid = 1, ///< Geometric MG + kFastRelaxation = 2 ///< Spectral (TODO) + }; + + ///< Enumeration of Cycles Type + enum CycleType { + kVCycle = 0, ///< V Cycle + kWCycle = 1, ///< W Cycle (TODO) + kFCycle = 2 ///< Full Cycle + }; + + ///< Fine -> Coarse Grid transfer operator types + enum GridTransferType { + kHalf = 0, ///< Half weighting + kFull = 1, ///< Full weighting + }; + + ///< Smoothing (Relax) operator types + enum RelaxType { + kJacobi = 0, ///< Jacobi (5 Stencil 2D, 7 Stencil 3D_ + kWeightedJacobi = 1, ///< (TODO) + kGaussSeidel = 2 ///< Gauss Seidel 2D (2 Color, 5 Stencil), 3D (7 Stencil) + }; + + ///< Coarse -> fine operator types (TODO: Interp and Restrict in one packet, just one enumeration) + enum InterpType { + kHalfInterp = 0, ///< Half bi linear interpolation + kFullInterp = 1 ///< Full bi linear interpolation + }; + + ///< Parameters choice for MultiGrid algorithm + struct MGParameters { + bool isFull3D; ///< TRUE: full coarsening, FALSE: semi coarsening + CycleType cycleType; ///< cycleType follow CycleType + GridTransferType gtType; ///< gtType grid transfer type follow GridTransferType + RelaxType relaxType; ///< relaxType follow RelaxType + int gamma; ///< number of iteration at coarsest level + int nPre; ///< number of iteration for pre smoothing + int nPost; ///< number of iteration for post smoothing + int nMGCycle; ///< number of multi grid cycle (V type) + int maxLoop; ///< the number of tree-deep of multi grid + + + // default values + MGParameters() { + isFull3D = false; + cycleType = kFCycle; + gtType = kFull; // default full + relaxType = kGaussSeidel; // default relaxation method + nPre = 2; + nPost = 2; + nMGCycle = 200; + maxLoop = 6; + + } + }; + + + + static const float fgkTPCZ0; ///< nominal gating grid position + static const float fgkIFCRadius; ///< Mean Radius of the Inner Field Cage ( 82.43 min, 83.70 max) (cm) + static const float fgkOFCRadius; ///< Mean Radius of the Outer Field Cage (252.55 min, 256.45 max) (cm) + static const float fgkZOffSet; ///< Offset from CE: calculate all distortions closer to CE as if at this point + static const float fgkCathodeV; ///< Cathode Voltage (volts) + static const float fgkGG; ///< Gating Grid voltage (volts) + static const float fgkdvdE; ///< [cm/V] drift velocity dependency on the E field (from Magboltz for NeCO2N2 at standard environment) + static const float fgkEM; ///< charge/mass in [C/kg] + static const float fgke0; ///< vacuum permittivity [A·s/(V·m)] + + static float fgExactErr; ///< Error tolerated + static float fgConvergenceError; ///< Error tolerated + int fIterations; ///< number of maximum iteration + MGParameters fMgParameters; ///< parameters multi grid + + void SetExactSolution(float *exactSolution, const int fPhiSlices); + void SetCycleType(PoissonSolver3DCylindricalGPU::CycleType cycleType) { + fMgParameters.cycleType = cycleType; + } + + StrategyType fStrategy; ///< strategy used default multiGrid + PoissonSolver3DCylindricalGPU(); + PoissonSolver3DCylindricalGPU(int nRRow, int nZColumn, int nPhiSlice); + PoissonSolver3DCylindricalGPU(const char *name, const char *title); + virtual ~PoissonSolver3DCylindricalGPU(); + void PoissonSolver3D(float *matricesV, float *matricesChargeDensities, int nRRow, int nZColumn, + int phiSlice, int maxIterations, int symmetry); + + + // setter and getter + void SetStrategy(StrategyType strategy) {fStrategy = strategy;} + StrategyType GetStrategy() { return fStrategy; } + void SetExactSolution(float *exactSolution,int nRRow, int nZColumn, int phiSlice); + float *fExactSolutionF; +private: + PoissonSolver3DCylindricalGPU(const PoissonSolver3DCylindricalGPU &); // not implemented + PoissonSolver3DCylindricalGPU &operator=(const PoissonSolver3DCylindricalGPU &); // not implemented + void PoissonMultiGrid3D2D(float *VPotential, float *RhoChargeDensities, int nRRow, + int nZColumn, int phiSlice, int symmetry); + + + + // store potential and charge + float * fVPotential; //-> hold potential in an object of tmatrixd + float * fRhoCharge; //-> pointer to an object of tmatrixd for storing charge + int fNRRow; + int fNZColumn; + int fPhiSlice; + // error single precision for cuda-based + float *fErrorConvF; + float *fErrorExactF; + bool fExactPresent; + float fMaxExact; +}; + +#endif diff --git a/kernel/CMakeLists.txt.aliroot b/kernel/CMakeLists.txt.aliroot new file mode 100644 index 0000000..eb62d07 --- /dev/null +++ b/kernel/CMakeLists.txt.aliroot @@ -0,0 +1,37 @@ +set(SRCS + ${SCGPUDIR}/interface/AliTPCPoissonSolverMem.cxx +# ${CAGPU_O} + ) + +# Headers from sources +set(CINTHDRS + ${SCGPUDIR}/interface/AliTPCPoissonSolverMem.h + ${SCGPUDIR}/interface/AliCudaLinkDef.h + ) + +set(HDRS + ) + +# Generate the dictionary +# It will create G_ARG1.cxx and G_ARG1.h / ARG1 = function first argument +get_directory_property(incdirs INCLUDE_DIRECTORIES) +generate_dictionary("${MODULE}" "" "${CINTHDRS}" "${incdirs}") + +# Generate the ROOT map +# Dependecies +generate_rootmap("${MODULE}" "" "") + +# Add a library to the project using the specified source files +add_library_tested(${MODULE} SHARED ${SRCS} G__${MODULE}.cxx) +#CUDA run-time and driver +target_link_libraries(${MODULE} ${CUDA_LIBRARIES} ${LIBCUDA_SO_PATH} AliTPCSpaceCharge3DDriftLine) + +# Additional compilation flags +set_target_properties(${MODULE} PROPERTIES COMPILE_FLAGS "") + +# Installation +install(TARGETS ${MODULE} + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib) + +install(FILES ${HDRS} DESTINATION include) diff --git a/gpulib/PoissonSolver3DGPU.cu b/kernel/PoissonSolver3DGPU.cu similarity index 100% rename from gpulib/PoissonSolver3DGPU.cu rename to kernel/PoissonSolver3DGPU.cu diff --git a/gpulib/PoissonSolver3DGPU.h b/kernel/PoissonSolver3DGPU.h similarity index 99% rename from gpulib/PoissonSolver3DGPU.h rename to kernel/PoissonSolver3DGPU.h index 3d6c294..6995a10 100644 --- a/gpulib/PoissonSolver3DGPU.h +++ b/kernel/PoissonSolver3DGPU.h @@ -4,7 +4,6 @@ #include #include #include - #include #include