fixes on the numa stuff (a1f0f5e7) · Commits · Claudio Gheller / HPC_Imaging

allvars.h

+7 −0

Original line number	Diff line number	Diff line
		/* file to store global variables*/


		#if defined(__STDC__)
		# if (__STDC_VERSION__ >= 199901L)
		# define _XOPEN_SOURCE 700
		# endif
		#endif

		#include <stdio.h>
		#include <stdlib.h>
		#include <string.h>

gridding.c

+8 −6

Original line number	Diff line number	Diff line
		@@ -242,7 +242,7 @@ void gridding_data(){

		// for every task, gridss coincides with the
		// that can be avoided if shared window coincides with gridss
		memcpy(Me.win.ptr+isector*sizeof(gridss), gridss, sizeof(gridss));
		memcpy(Me.win.ptr+isectorsize_of_grid, gridss, size_of_gridsizeof(double));


		reduce( isector, target_rank ); // here the reduce is performed within every host
		@@ -275,7 +275,7 @@ void gridding_data(){
		if ( global_rank == target_rank)
		{
		MPI_Send( &sector_rank, 1, MPI_INT, 0, 0, Sector_Comm);
		memcpy(grid, Me.swins[Me.Rank[myHOST]].ptr+isector*sizeof(gridss), sizeof(grid));
		memcpy(grid, Me.swins[Me.Rank[myHOST]].ptr+isectorsize_of_gridsizeof(double), size_of_grid * sizeof(double));
		}

		if( sector_rank == 0 )
		@@ -451,6 +451,8 @@ void reduce( int sector, int target_rank )

		(int)(Me.win_ctrl.ptr) = -1;

		double my_base = ((double)Me.win.ptr)+sector*size_of_grid;

		for(int l = 0; l < max_level; l++)
		{
		int threshold = 1 << (1+l);
		@@ -462,9 +464,9 @@ void reduce( int sector, int target_rank )
		// sleep 5 usec if the source target is not ready
		NSLEEP( 5000 );

		double source_base = ((double)Me.swins[source].ptr)+sector*size_of_grid;
		for(int j = 0; j < size_of_grid; j++)
		((double)Me.swins[local_rank].ptr+sector*sizeof(gridss)+j) +=
		((double)Me.swins[source].ptr+sector*sizeof(gridss)+j);
		(my_base + j) += (source_base + j);

		(int)(Me.win_ctrl.ptr) = l;
		}

init.c

+64 −56

Original line number	Diff line number	Diff line
		@@ -31,7 +31,7 @@ void init(int index)
		yaxis = local_grid_size_y;

		#ifdef USE_MPI
		init_numa( global_rank, size, &MYMPI_COMM_WORLD, &Me );
		numa_init( global_rank, size, &MYMPI_COMM_WORLD, &Me );

		if(global_rank == 0)
		printf("\nTask %d sees %d topology levels\n", global_rank, Me.MAXl);
		@@ -123,13 +123,14 @@ void op_filename() {
		MPI_Bcast(&out, sizeof(struct op), MPI_BYTE, 0, MPI_COMM_WORLD);
		#endif

		return;
		}

		void read_parameter_file(char *fname)
		{
		if(global_rank == 0)
		{
		if(file.pFile = fopen (fname,"r"))
		if( (file.pFile = fopen (fname,"r")) != NULL )
		{
		char buf1[30], buf2[100], buf3[30], num[30];
		int i = 1;
		@@ -275,7 +276,7 @@ void readMetaData(char fileLocal[1000]) {

		if(global_rank == 0)
		{
		if(file.pFile = fopen (fileLocal,"r"))
		if( (file.pFile = fopen (fileLocal,"r")) != NULL )
		{
		fscanf(file.pFile,"%ld",&metaData.Nmeasures);
		fscanf(file.pFile,"%ld",&metaData.Nvis);
		@@ -293,7 +294,8 @@ void readMetaData(char fileLocal[1000]) {
		}
		else
		{
		printf("error opening meta file");
		printf("error opening meta file %s\n",
		fileLocal);
		exit(1);
		}
		}
		@@ -384,7 +386,7 @@ void readData() {
		}

		fileName(datapath, in.ufile);
		if(file.pFile = fopen (filename,"rb"))
		if( (file.pFile = fopen (filename,"rb")) != NULL )
		{
		fseek (file.pFile,startrow*sizeof(double),SEEK_SET);
		fread(data.uu,metaData.Nmeasures*sizeof(double),1,file.pFile);
		@@ -392,12 +394,13 @@ void readData() {
		}
		else
		{
		printf("error opening ucoord file");
		printf("error opening ucoord file %s\n",
		filename );
		exit(1);
		}

		fileName(datapath, in.vfile);
		if(file.pFile = fopen (filename,"rb"))
		if( (file.pFile = fopen (filename,"rb")) != NULL )
		{
		fseek (file.pFile,startrow*sizeof(double),SEEK_SET);
		fread(data.vv,metaData.Nmeasures*sizeof(double),1,file.pFile);
		@@ -405,12 +408,13 @@ void readData() {
		}
		else
		{
		printf("error opening vcoord file");
		printf("error opening vcoord file %s\n",
		filename);
		exit(1);
		}

		fileName(datapath, in.wfile);
		if(file.pFile = fopen (filename,"rb"))
		if( (file.pFile = fopen (filename,"rb")) != NULL )
		{
		fseek (file.pFile,startrow*sizeof(double),SEEK_SET);
		fread(data.ww,metaData.Nmeasures*sizeof(double),1,file.pFile);
		@@ -418,12 +422,13 @@ void readData() {
		}
		else
		{
		printf("error opening wcoord file");
		printf("error opening wcoord file %s\n",
		filename);
		exit(1);
		}

		fileName(datapath, in.weightsfile);
		if(file.pFile = fopen (filename,"rb"))
		if( (file.pFile = fopen (filename,"rb")) != NULL)
		{
		fseek (file.pFile,startrowmetaData.polarisationssizeof(float),SEEK_SET);
		fread(data.weights,(metaData.Nweights)*sizeof(float),1,file.pFile);
		@@ -431,12 +436,13 @@ void readData() {
		}
		else
		{
		printf("error opening weights file");
		printf("error opening weights file %s\n",
		filename);
		exit(1);
		}

		fileName(datapath, in.visrealfile);
		if(file.pFile = fopen (filename,"rb"))
		if((file.pFile = fopen (filename,"rb")) != NULL )
		{
		fseek (file.pFile,startrowmetaData.freq_per_chanmetaData.polarisations*sizeof(float),SEEK_SET);
		fread(data.visreal,metaData.Nvis*sizeof(float),1,file.pFile);
		@@ -444,12 +450,13 @@ void readData() {
		}
		else
		{
		printf("error opening visibilities_real file");
		printf("error opening visibilities_real file %s\n",
		filename);
		exit(1);
		}

		fileName(datapath, in.visimgfile);
		if(file.pFile = fopen (filename,"rb"))
		if( (file.pFile = fopen (filename,"rb")) != NULL )
		{
		fseek (file.pFile,startrowmetaData.freq_per_chanmetaData.polarisations*sizeof(float),SEEK_SET);
		fread(data.visimg,metaData.Nvis*sizeof(float),1,file.pFile);
		@@ -457,7 +464,8 @@ void readData() {
		}
		else
		{
		printf("error opening visibilities_img file");
		printf("error opening visibilities_img file %s\n",
		filename);
		exit(1);
		}

main.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -52,7 +52,7 @@ int main(int argc, char * argv[])

		if ( param.num_threads == 0 )
		{
		fprintf(stderr, "Usage: %s number_of_OMP_Threads \n", param.num_threads);
		fprintf(stderr, "Usage: %d number_of_OMP_Threads \n", param.num_threads);
		exit(1);
		}

numa.c

+56 −29

Original line number	Diff line number	Diff line
		@@ -4,13 +4,13 @@
		#include <unistd.h>
		#include <limits.h>


		map_t Me;
		MPI_Comm COMM[HLEVELS];

		char *LEVEL_NAMES[HLEVELS] = {"NUMA", "ISLAND", "myHOST", "HOSTS", "WORLD"};

		MPI_Aint win_host_master_size = 0;
		MPI_Aint win_host__size = 0;

		MPI_Aint win_ctrl_hostmaster_size;
		MPI_Win win_ctrl_hostmaster;
		@@ -23,19 +23,19 @@ int win_hostmaster_disp;
		void *win_hostmaster_ptr;


		int build_numa_mapping( int, int, MPI_Comm , map_t );
		int map_hostnames( MPI_Comm , int, int, map_t );
		int numa_build_mapping( int, int, MPI_Comm , map_t );
		int numa_map_hostnames( MPI_Comm , int, int, map_t );
		int get_cpu_id( void );
		int compare_string_int_int( const void , const void );


		int init_numa( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		int numa_init( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		{

		/*
		* build up the numa hierarchy
		*/
		build_numa_mapping( global_rank, size, MYWORLD, Me );
		numa_build_mapping( global_rank, size, MYWORLD, Me );

		/*
		* initialize the persistent shared windows
		@@ -70,21 +70,44 @@ int init_numa( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		MPI_Win_shared_query( win_ctrl_hostmaster, 0, &(win_ctrl_hostmaster_size),
		&win_ctrl_hostmaster_disp, &win_ctrl_hostmaster_ptr );


		return 0;
		}


		int numa_allocate_shared_windos( map_t *Me, MPI_Aint size, MPI_Aint host_size )
		{

		int SHMEMl = Me->SHMEMl;
		MPI_Info winfo;

		MPI_Info_create(&winfo);
		MPI_Info_set(winfo, "alloc_shared_noncontig", "true");

		// -----------------------------------
		// initialize the data windows
		// -----------------------------------
		MPI_Aint win_host_size;

		if( host_size == 0 )
		win_hostmaster_size = WIN_HOST_MASTER_SIZE_DFLT10241024;
		MPI_Aint win_host_size = WIN_HOST_SIZE_DFLT10241024;
		else
		win_hostmaster_size = host_size;

		if( size == 0 )
		win_host_size = WIN_HOST_SIZE_DFLT10241024;
		else
		win_host_size = size;


		Me->win.size = win_host_size;
		MPI_Win_allocate_shared(Me->win.size, 1, winfo, *Me->COMM[SHMEMl], &(Me->win.ptr), &(Me->win.win));

		wsize = ( Me->Rank[SHMEMl] == 0 ? win_hostmaster_size : 0);
		MPI_Aint wsize = ( Me->Rank[SHMEMl] == 0 ? win_hostmaster_size : 0);
		MPI_Win_allocate_shared(wsize, 1, winfo, *Me->COMM[SHMEMl], &win_hostmaster_ptr, &win_hostmaster);

		Me->swins = (win_t)malloc(Me->Ntasks[SHMEMl]sizeof(win_t) );
		Me->swins[Me->Rank[SHMEMl]] = Me->win;
		// Me->swins = (win_t)malloc(Me->Ntasks[SHMEMl]sizeof(win_t));
		// get the addresses of all the windows from my siblings
		// at my shared-memory level
		@@ -96,10 +119,10 @@ int init_numa( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		if( Me->Rank[SHMEMl] != 0 )
		MPI_Win_shared_query( win_hostmaster, 0, &(win_hostmaster_size), &win_hostmaster_disp, &win_hostmaster_ptr );

		return 0;
		}


		int shutdown_numa( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		int numa_shutdown( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		{
		// free every shared memory and window
		//
		@@ -114,9 +137,11 @@ int shutdown_numa( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		//
		// ...

		return 0;

		}

		int build_numa_mapping( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		int numa_build_mapping( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		{
		COMM[WORLD] = *MYWORLD;

		@@ -130,7 +155,7 @@ int build_numa_mapping( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		// that is needed to build the communicator
		// among the masters of each host
		//
		map_hostnames( &COMM[WORLD], Rank, Size, Me );
		numa_map_hostnames( &COMM[WORLD], Rank, Size, Me );


		Me->MAXl = ( Me->Nhosts > 1 ? HOSTS : myHOST );
		@@ -145,6 +170,19 @@ int build_numa_mapping( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		Me->Rank[myHOST] = Rank;
		Me->Ntasks[myHOST] = Size;

		// with the following gathering we build-up the mapping Ranks_to_hosts, so that
		// we know which host each mpi rank (meaning the original rank) belongs to
		//

		MPI_Allgather( &Me->myhost, sizeof(Me->myhost), MPI_BYTE,
		Me->Ranks_to_host, sizeof(Me->myhost), MPI_BYTE, COMM[WORLD] );

		Me -> Ranks_to_myhost = (int)malloc(Me->Ntasks[myHOST]sizeof(int));
		MPI_Allgather( &global_rank, sizeof(global_rank), MPI_BYTE,
		Me->Ranks_to_myhost, sizeof(global_rank), MPI_BYTE, *Me->COMM[myHOST]);



		// --- create the communicator for the
		// masters of each host
		//
		@@ -220,7 +258,7 @@ int build_numa_mapping( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		}


		int map_hostnames( MPI_Comm *MY_WORLD, // the communicator to refer to
		int numa_map_hostnames( MPI_Comm *MY_WORLD, // the communicator to refer to
		int Rank, // the initial rank of the calling process in MYWORLD
		int Ntasks, // the number of tasks in MY_WORLD
		map_t *me) // address of the info structure for the calling task
		@@ -289,17 +327,6 @@ int map_hostnames( MPI_Comm *MY_WORLD, // the communicator to refer to
		}
		me->Nhosts++;

		// with the following gathering we build-up the mapping Ranks_to_hosts, so that
		// we know which host each mpi rank (meaning the original rank) belongs to
		//

		MPI_Allgather( &me->myhost, sizeof(me->myhost), MPI_BYTE,
		me->Ranks_to_host, sizeof(me->myhost), MPI_BYTE, *MY_WORLD );

		me -> Ranks_to_myhost = (int)malloc(me->Ntasks[myHOST]sizeof(int));
		MPI_Allgather( &global_rank, sizeof(global_rank), MPI_BYTE,
		me->Ranks_to_myhost, sizeof(global_rank), MPI_BYTE, *me->COMM[myHOST]);

		free( alldata );

		return me->Nhosts;
		@@ -370,7 +397,7 @@ int get_cpu_id( void )

		#else

		unsigned val;
		int val;
		if ( read_proc__self_stat( CPU_ID_ENTRY_IN_PROCSTAT, &val ) == -1 )
		return -1;