Merge branch 'revamp' of https://www.ict.inaf.it/gitlab/claudio.gheller/hpc_imaging into revamp (dcc35bbe) · Commits · Claudio Gheller / HPC_Imaging

gridding.c

+18 −25

Original line number	Diff line number	Diff line
		@@ -250,7 +250,7 @@ void gridding_data()
		// that can be avoided if shared window coincides with gridss

		TAKE_TIME(twt, tpr);
		memcpy(Me.win.ptr+isectorsize_of_grid, gridss, size_of_gridsizeof(double));
		memcpy(Me.win.ptr, gridss, size_of_grid*sizeof(double));
		ADD_TIME(mmove, twt, tpr);

		dprintf(1, global_rank, 0, "reducing sector %ld..\n", isector);
		@@ -439,10 +439,9 @@ void write_gridded_data()
		}


		#define NSLEEP( T ) {struct timespec tsleep={0, (T)}; nanosleep(&tsleep, NULL); }

		void reduce( int sector, int target_rank )
		{
		MPI_Barrier(*(Me.COMM[myHOST]));

		int local_rank = Me.Rank[myHOST];
		int target_rank_on_myhost = -1;
		@@ -458,15 +457,14 @@ void reduce( int sector, int target_rank )
		while( Me.Ranks_to_myhost[target_rank_on_myhost] != target_rank )
		target_rank_on_myhost++;

		dprintf(2, Me.Rank[myHOST], 0,
		"[SEC %d] swapping Host master with target rank %d (%d)\n",
		sector, target_rank, target_rank_on_myhost);


		if( target_rank_on_myhost > 0 )
		// the target is not the task that already has rank 0
		// on my host
		{
		dprintf(2, Me.Rank[myHOST], 0,
		"[SEC %d] swapping Host master with target rank %d (%d)\n",
		sector, target_rank, target_rank_on_myhost);

		if( local_rank == 0 )
		local_rank = target_rank_on_myhost;
		@@ -488,37 +486,32 @@ void reduce( int sector, int target_rank )
		while( (1<< (++max_level) ) < Me.Ntasks[myHOST] );

		(int)(Me.win_ctrl.ptr) = -1;

		double my_base = ((double)Me.win.ptr)+sector*size_of_grid;
		MPI_Win_fence( 0, Me.win_ctrl.win);

		for(int l = 0; l < max_level; l++)
		{
		int threshold = 1 << (1+l);

		MPI_Win_fence( MPI_MODE_NOSTORE, Me.win_ctrl.win);
		if( local_rank % threshold == 0)
		{
		int source = local_rank + (1<<l);
		dprintf(2, 0, 0,
		"[SEC %d] task %d (%d) getting data from task %d at level %d\n",
		sector, local_rank, Me.Rank[myHOST], source, l );

		while( (int)(Me.scwins[source].ptr) < l )
		// sleep 5 usec if the source target is not ready
		NSLEEP( 5000 );

		double source_base = ((double)Me.swins[source].ptr)+sector*size_of_grid;
		for(int j = 0; j < size_of_grid; j++)
		(my_base + j) += (source_base + j);
		if( source < Me.Ntasks[myHOST] )
		{
		double * restrict my_base = ((double*)Me.win.ptr);
		double * my_end = my_base + size_of_grid;
		double * restrict source_base = ((double*)Me.swins[source].ptr);
		for( ; my_base < my_end; my_base++, source_base++)
		my_base += source_base;
		}

		(int)(Me.win_ctrl.ptr) = l;
		MPI_Win_fence( 0, Me.win_ctrl.win);
		}
		else
		{
		dprintf(2, 0, 0,
		"[SEC %d] task %d (%d) signaling that level %d is done\n",
		sector, local_rank, Me.Rank[myHOST], l );

		(int)(Me.win_ctrl.ptr) = l;
		MPI_Win_fence( 0, Me.win_ctrl.win);
		}
		}

init.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -368,7 +368,7 @@ void allocate_memory() {
		gridss_real = (double*) calloc(size_of_grid/2,sizeof(double));
		gridss_img = (double*) calloc(size_of_grid/2,sizeof(double));

		numa_allocate_shared_windows( &Me, size_of_gridsizeof(double)1.1, size_of_gridsizeof(double)1.1);
		numa_allocate_shared_windows( &Me, size_of_gridsizeof(double)1.1, 0 );

		// Create destination slab
		grid = (double*) calloc(size_of_grid,sizeof(double));

numa.c

+8 −9

Original line number	Diff line number	Diff line
		@@ -35,7 +35,7 @@ int numa_init( int Rank, int Size, MPI_Comm MYWORLD, map_t Me )
		/*
		* build up the numa hierarchy
		*/
		numa_build_mapping( global_rank, size, MYWORLD, Me );
		numa_build_mapping( Rank, Size, MYWORLD, Me );

		/*
		* initialize the persistent shared windows
		@@ -91,12 +91,12 @@ int numa_allocate_shared_windows( map_t *me, MPI_Aint size, MPI_Aint host_size
		// -----------------------------------
		MPI_Aint win_host_size;

		if( host_size == 0 )
		if( host_size < 0 )
		win_hostmaster_size = WIN_HOST_MASTER_SIZE_DFLT10241024;
		else
		win_hostmaster_size = host_size;

		if( size == 0 )
		if( size < 0 )
		win_host_size = WIN_HOST_SIZE_DFLT10241024;
		else
		win_host_size = size;
		@@ -180,8 +180,8 @@ int numa_build_mapping( int Rank, int Size, MPI_Comm MYWORLD, map_t me )
		me->Ranks_to_host, sizeof(me->myhost), MPI_BYTE, COMM[WORLD] );

		me -> Ranks_to_myhost = (int)malloc(me->Ntasks[myHOST]sizeof(int));
		MPI_Allgather( &global_rank, sizeof(global_rank), MPI_BYTE,
		me->Ranks_to_myhost, sizeof(global_rank), MPI_BYTE, *me->COMM[myHOST]);
		MPI_Allgather( &Rank, sizeof(Rank), MPI_BYTE,
		me->Ranks_to_myhost, sizeof(Rank), MPI_BYTE, *me->COMM[myHOST]);



		@@ -296,7 +296,7 @@ int numa_map_hostnames( MPI_Comm *MY_WORLD, // the communicator to refer to
		hostname_rank_t alldata = (hostname_rank_t)calloc( Ntasks, sizeof(hostname_rank_t) );

		mydata.rank = Rank;
		sprintf( mydata.hostname, "%s", myhostname);
		snprintf( mydata.hostname, maxhostlen, "%s", myhostname);

		MPI_Allgather( &mydata, sizeof(hostname_rank_t), MPI_BYTE, alldata, sizeof(hostname_rank_t), MPI_BYTE, *MY_WORLD );

		@@ -317,7 +317,6 @@ int numa_map_hostnames( MPI_Comm *MY_WORLD, // the communicator to refer to
		// --- count how many diverse hosts we have, and register each rank to its host, so that
		// we can alway find all the tasks with their original rank


		char *prev = alldata[0].hostname;
		for ( int R = 0; R < Ntasks; R++ )
		{
		@@ -350,7 +349,7 @@ int compare_string_int_int( const void A, const void B )
		static int str_len = 0;
		if ( A == NULL )
		{
		str_len = (int)B + 1;
		str_len = (int)B;
		return 0;
		}