Commit 82ed9b5a authored by David Goz's avatar David Goz 😴
Browse files

mpi/comp_comm_io MPI I/O added

parent ab66905e

jacobi/mpi/comp_comm/src/jacobi_2D_mpi_comp_comm.c

+27 −10
Original line number Diff line number Diff line
@@ -136,7 +136,7 @@ int main(int argc, char **argv) 
  

  /************************************************************************************************************/

  /* 2D MPI-cartesian decomposition:

     the grids are distributed across the MPI processes.

     the grids are distributed across the MPI processes. */



  /* get the reminder, i.e. take into account uneven

     decomposition of points among the processes    */
@@ -593,7 +593,6 @@ void get_domains(MyData **const buffer, 
  int *recvcounts = NULL;

  int *displs = NULL;

  MyData *recvbuf = NULL;

  MyData *sendbuf = NULL;

  

  if (ThisTask->rank == MASTERTASK)

    {
@@ -618,19 +617,37 @@ void get_domains(MyData **const buffer, 
      assert(recvbuf != NULL);

    } /* MASTERTASK */



  /* every process sets up the sendbuf */

  sendbuf = (MyData *)malloc(ThisTask->domain.dim[X] * ThisTask->domain.dim[Y] * sizeof(*sendbuf));

  assert(sendbuf != NULL);

  /* store the actual grid owned by each process into sendbuf */

  for (int row=0 ; row<ThisTask->domain.dim[X] ; row++)

    memcpy(&sendbuf[row * ThisTask->domain.dim[Y]], &buffer[ThisTask->domain.local_start[X] + row][ThisTask->domain.local_start[Y]], (ThisTask->domain.dim[Y] * sizeof(MyData)));

  /*************************************** CUSTOM subarray ******************************************************************************/

  /* Do not do that !!!                                                                                                                 */

  /* Use instead MPI_Type_create_subarray                                                                                               */

  /*                                                                                                                                    */

  /* MyData *subarray = (MyData *)malloc(ThisTask->domain.dim[X] * ThisTask->domain.dim[Y] * sizeof(*subarray));                        */

  /* for (int row=0 ; row<ThisTask->domain.dim[X] ; row++)                                                                              */

  /*   memcpy(&sendbuf[row * ThisTask->domain.dim[Y]], &buffer[ThisTask->domain.local_start[X] + row][ThisTask->domain.local_start[Y]], */

  /*            (ThisTask->domain.dim[Y] * sizeof(MyData)));                                                                            */

  /*                                                                                                                                    */

  /* Perform MPI_Gatherv collective...                                                                                                  */

  /*                                                                                                                                    */

  /* MPI_Gatherv(sendbuf, (ThisTask->domain.dim[X] * ThisTask->domain.dim[Y]), MPI_MyDatatype,                                          */

  /*             recvbuf, recvcounts, displs, MPI_MyDatatype,                                                                           */

  /*             MASTERTASK, ThisTask->comm2d);                                                                                         */

  /**************************************************************************************************************************************/

  

  /* every process sets up the subarray */

  MPI_Datatype subarray;

  const int subsize[NDIM] = {ThisTask->domain.dim[X], ThisTask->domain.dim[Y]};

  const int bigsize[NDIM] = {subsize[X] + 2 *NGHOST, subsize[Y] + 2 *NGHOST};

  const int start[NDIM] = {NGHOST, NGHOST};

  MPI_Type_create_subarray(NDIM, bigsize, subsize, start, MPI_ORDER_C, MPI_MyDatatype, &subarray);

  MPI_Type_commit(&subarray);

  

  /* perform the MPI collective */

  MPI_Gatherv(sendbuf, (ThisTask->domain.dim[X] * ThisTask->domain.dim[Y]), MPI_MyDatatype,

  MPI_Gatherv(&buffer[0][0], 1, subarray,

	      recvbuf, recvcounts, displs, MPI_MyDatatype,

	      MASTERTASK, ThisTask->comm2d);



  free(sendbuf);

  MPI_Type_free(&subarray);



  if (ThisTask->rank == MASTERTASK)

    {

      free(recvcounts);

jacobi/mpi/comp_comm_io/Makefile

0 → 100644
+61 −0
Original line number Diff line number Diff line 
#######################################################################

# Author: David Goz (david.goz@inaf.it)                               #

# June  2024                                                          #

#######################################################################

#

# To see all the compilation options

# $ make info

#######################################################################



# make.def defines how the application is compiled

include make.def

include make_mpi_path

#######################################################################



.PHONY: info mpi valgrind_memcheck valgrind_callgrind valgrind_cachegrind clean



info:

	@echo ' '

	@echo '-----------------------------------------------------------------------------------------'

	@echo '$$ make mpi                 ---> compile the mpi application                             '

	@echo '$$ make debug               ---> compile the mpi application for debugging               '

	@echo '$$ make valgrind_memcheck   ---> run the mpi application using Valgrind under Memcheck   '

	@echo '$$ make valgrind_callgrind  ---> run the mpi application using Valgrind under Callgrind  '

	@echo '$$ make valgrind_cachegrind ---> run the mpi application using Valgrind under Cachegrind '

	@echo '$$ make clean               ---> clean up all                                            '

	@echo '$$ make info                ---> get make info                                           '

	@echo '-----------------------------------------------------------------------------------------'

	@echo ' '



mpi: $(PROG)



debug: $(PROG_DEBUG)



valgrind_memcheck: $(PROG_MEMCHECK)

	@echo 'oooOOO... valgrind_memcheck ...OOOooo'

	mpirun -n 4 valgrind --tool=memcheck -s --default-suppressions=yes --log-file=valgrind_memcheck_log_%p.txt ./$< 9 2

	@echo 'oooOOO... valgrind_memcheck ...OOOooo'



valgrind_callgrind: $(PROG_CALLGRIND)

	@echo 'oooOOO... valgrind_callgrind ...OOOooo'

	mpirun -n 4 valgrind --tool=callgrind --dump-instr=yes --collect-jumps=yes --log-file=valgrind_callgrind_log_.%p.txt ./$< 9 2

	@echo ' '

	@echo 'To generate a function-by-function summary from the profile data file:'

	@echo '$$ callgrind_annotate --auto=yes callgrind.out.<pid> | less'

	@echo '(kcachegrind is required in order to visualize the output using the GUI)'



valgrind_cachegrind: $(PROG_CACHEGRIND)

	@echo 'oooOOO... valgrind_cachegrind ...OOOooo'

	mpirun -n 4 valgrind --tool=cachegrind --log-file=valgrind_cachegrind_log_.%p.txt ./$< 9 2

	@echo '$$ cg_annotate --auto=yes cachegrind.out.<pid> | less'

	@echo '(kcachegrind is required in order to visualize the output using the GUI)'

	@echo 'oooOOO... valgrind_cachegrind ...OOOooo'



clean:

	rm -f *~ .*~ ./src/*~ ./src/*# ./include/*~ ./include/*# *~

	rm -f $(PROG) $(PROG_DEBUG) $(PROG_MEMCHECK) $(PROG_CALLGRIND) $(PROG_CACHEGRIND)

	rm -f valgrind_*.txt

	rm -f cachegrind.out.*

	rm -f callgrind.*

	rm -f *bin

	rm -f jacobi_mpi_comp_comm_*

jacobi/mpi/comp_comm_io/include/allvars.h

0 → 100644
+20 −0
Original line number Diff line number Diff line 
#pragma once



#define NGHOST   1

#define NDIM     2 /* 2D */

#define X        0

#define Y        1

#define TOL      1.e-5



#include <mpi.h>



#define MASTERTASK 0



#if defined(SINGLE_PRECISION)

typedef float MyData;

#define MPI_MyDatatype MPI_FLOAT_PRECISION

#else

/* double precision is assumed by default */

typedef double MyData;

#define MPI_MyDatatype     MPI_DOUBLE_PRECISION

#endif /* SINGLE_PRECISION */

jacobi/mpi/comp_comm_io/include/tools.h

0 → 100644
+18 −0
Original line number Diff line number Diff line 
#pragma once



#include "allvars.h"

#include <assert.h>

#include <sys/time.h>

#include <time.h>

#include <sys/stat.h>

#include <stdio.h>

#include <stdlib.h>



/* function prototypes */

MyData **Allocate_2DdblArray(const int nx, const int ny);

void Show_2DdblArray(      MyData **const A,

                     const int            nx,

                     const int            ny,

                     const char *const    string);



double seconds(void);

jacobi/mpi/comp_comm_io/make.def

0 → 100644
+45 −0
Original line number Diff line number Diff line 
CC     = mpicc

CFLAGS = -Wall -Wextra -march=native

LIBS   = -lm -lmpi



SYSTYPE = $(strip $(shell uname -n))



PROG            = jacobi_mpi_comp_comm_$(SYSTYPE)

PROG_DEBUG      = $(PROG)_DEBUG

PROG_MEMCHECK   = $(PROG)_MEMCHECK

PROG_CALLGRIND  = $(PROG)_CALLGRIND

PROG_CACHEGRIND = $(PROG)_CACHEGRIND



HEADERS       = $(wildcard ./include/*.h)

SOURCES       = $(wildcard ./src/*.c)

DEPENDENCIES  = $(SOURCES) $(HEADERS) Makefile



$(PROG): $(DEPENDENCIES)

	$(CC) $(CFLAGS) -O3 -I./include -I$(MPI_INC) $(SOURCES) -o $@ -L$(MPI_LIB) $(LIBS)

	@echo ' '

	@echo 'Program' $(PROG) 'compiled for' $(SYSTYPE) 'machine'

	@echo ' '



$(PROG_DEBUG): $(DEPENDENCIES)

	$(CC) $(CFLAGS) -Og -ggdb3 -DDEBUG -fno-omit-frame-pointer -I./include -I$(MPI_INC) $(SOURCES) -o $@ -L$(MPI_LIB) $(LIBS)

	@echo ' '

	@echo 'Program' $(PROG_DEBUG) 'compiled for' $(SYSTYPE) 'machine'

	@echo ' '



$(PROG_MEMCHECK): $(DEPENDENCIES)

	$(CC) $(CFLAGS) -Og -I./include -I$(MPI_INC) $(SOURCES) -o $@ -L$(MPI_LIB) $(LIBS)

	@echo ' '

	@echo 'Program' $(PROG_MEMCHECK) 'compiled for' $(SYSTYPE) 'machine'

	@echo ' '



$(PROG_CALLGRIND): $(DEPENDENCIES)

	$(CC) $(CFLAGS) -g -O3 -I./include -I$(MPI_INC) $(SOURCES) -o $@ -L$(MPI_LIB) $(LIBS)

	@echo ' '

	@echo 'Program' $(PROG_CALLGRIND) 'compiled for' $(SYSTYPE) 'machine'

	@echo ' '



$(PROG_CACHEGRIND): $(DEPENDENCIES)

	$(CC) $(CFLAGS) -g -O3 -I./include -I$(MPI_INC) $(SOURCES) -o $@ -L$(MPI_LIB) $(LIBS)

	@echo ' '

	@echo 'Program' $(PROG_CACHEGRIND) 'compiled for' $(SYSTYPE) 'machine'

	@echo ' '