Commit da2f017a authored by lykos98's avatar lykos98
Browse files

send_rcv change in all to all in exchange points

parent c00fd663

bb_plot.ipynb

+36 −24

File changed.

Preview size limit exceeded, changes collapsed.

src/tree/tree.c

+64 −57
Original line number Diff line number Diff line
@@ -1080,8 +1080,11 @@ void exchange_points(global_context_t* ctx, top_kdtree_t* tree) 
		points_per_proc[i] = points_per_proc[i] - points_per_proc[i - 1];

	}



	int* 	 rcvcount  = (int*)malloc(ctx -> world_size * sizeof(int));

	int* 	 displs    = (int*)malloc(ctx -> world_size * sizeof(int));

	int* rcv_count      = (int*)malloc(ctx -> world_size * sizeof(int));

	int* rcv_displs     = (int*)malloc(ctx -> world_size * sizeof(int));

    int* send_displs    = (int*)malloc(ctx -> world_size * sizeof(int)); 

    int* send_count     = points_per_proc;



	float_t* rcvbuffer = NULL;

	int tot_count = 0;
@@ -1090,39 +1093,32 @@ void exchange_points(global_context_t* ctx, top_kdtree_t* tree) 


    MPI_Barrier(ctx -> mpi_communicator);





	for(int rcv = 0; rcv < ctx -> world_size; ++rcv)

    /* TODO: change it to an all to all*/

    MPI_Alltoall(send_count, 1, MPI_INT, rcv_count, 1, MPI_INT, ctx -> mpi_communicator);

    rcv_displs[0] = 0;

    send_displs[0] = 0;

    for(int i = 1; i < ctx -> world_size; ++i) 

    {

		/* recieve the number of points to recieve from each proc */

		MPI_Gather(&(points_per_proc[rcv]), 1, MPI_INT, rcvcount, 1, MPI_INT, rcv, ctx -> mpi_communicator);

		float_t* send_buffer = ctx -> local_data + (ctx -> dims * partition_offset[rcv]);





		/* if I am the reciever recieve */

		if(rcv == ctx -> mpi_rank)

		{

			displs[0] = 0;

			for(int i = 1; i < ctx -> world_size; ++i) displs[i] = displs[i - 1] + rcvcount[i - 1];

        rcv_displs[i] = rcv_displs[i - 1] + rcv_count[i - 1];

        send_displs[i] = send_displs[i - 1] + send_count[i - 1];

    }



    /*multiply for number of elements */

    for(int i = 0; i < ctx -> world_size; ++i) 

    {

				displs[i]   = displs[i] * ctx -> dims;

				rcvcount[i] = rcvcount[i] * ctx -> dims;

				tot_count += rcvcount[i];

        send_displs[i]= send_displs[i] * ctx -> dims;

        send_count[i] = send_count[i] * ctx -> dims;



        rcv_displs[i]= rcv_displs[i] * ctx -> dims;

        rcv_count[i] = rcv_count[i] * ctx -> dims;

        tot_count += rcv_count[i];

    }

			//DB_PRINT("[RANK %d] is recieving %d elements %d points\n", rcv, tot_count, tot_count / ctx -> dims);



    rcvbuffer = (float_t*)malloc(tot_count * sizeof(float_t));



            DB_PRINT("Rank %d recieving: \t", ctx -> mpi_rank);

            for(int i = 0; i < ctx -> world_size; ++i)

            {

                DB_PRINT("%d:%d \t", i, rcvcount[i]);

            }

            DB_PRINT("\n");

		}

		MPI_Gatherv(send_buffer, ctx -> dims * points_per_proc[rcv], MPI_MY_FLOAT, rcvbuffer, rcvcount, displs, MPI_MY_FLOAT, rcv, ctx -> mpi_communicator);			

	}

    MPI_Alltoallv(  ctx -> local_data, send_count, send_displs, MPI_MY_FLOAT, 

                    rcvbuffer, rcv_count, rcv_displs, MPI_MY_FLOAT , 

                    ctx -> mpi_communicator);



	ctx -> local_n_points = tot_count / ctx -> dims; 

    int* ppp = (int*)malloc(ctx -> world_size * sizeof(int));
@@ -1160,8 +1156,8 @@ void exchange_points(global_context_t* ctx, top_kdtree_t* tree) 
	free(points_owners);

	free(points_per_proc);

	free(partition_offset);

	free(rcvcount);

	free(displs);

	free(rcv_count);

	free(rcv_displs);

}



static inline size_t local_to_global_idx(global_context_t* ctx, size_t local_idx)
@@ -1196,6 +1192,9 @@ void tree_walk( 
{



	if(root -> owner != -1 && root -> owner != ctx -> mpi_rank)

	{

        

        #pragma omp critical

        {

            /* put the leaf on the requests array */

            int owner = root -> owner;
@@ -1215,6 +1214,7 @@ void tree_walk( 
            local_idx_of_the_point[owner][idx] = point_idx;



            point_to_send_count[owner]++;

        }



	}

	else
@@ -1331,7 +1331,8 @@ void mpi_ngbh_search(global_context_t* ctx, datapoint_info_t* dp_info, top_kdtre 
	/* for each point walk the tree and find to which proc send data */

	/* actually compute intersection of ngbh radius of each point to node box */



    /* tree walk for each point */

    /* tree walk for each point can be optimized drasticaly*/

    #pragma omp parallel for

	for(int i = 0; i < ctx -> local_n_points; ++i)

	{

        /*
@@ -1502,6 +1503,7 @@ void mpi_ngbh_search(global_context_t* ctx, datapoint_info_t* dp_info, top_kdtre 
        heap_sort(&(dp_info[i].ngbh));

    }



    MPI_DB_PRINT("Writing ngbh to files\n");

    char ngbh_out[80];

    sprintf(ngbh_out, "./bb/rank_%d.ngbh",ctx -> mpi_rank);

    FILE* file = fopen(ngbh_out,"w");
@@ -1511,6 +1513,9 @@ void mpi_ngbh_search(global_context_t* ctx, datapoint_info_t* dp_info, top_kdtre 
    }

    fclose(file);



    MPI_Barrier(ctx -> mpi_communicator);

    





    

	for(int i = 0; i < ctx -> world_size; ++i)
@@ -1567,6 +1572,7 @@ void build_local_tree(global_context_t* ctx, kdtree_v2* local_tree) 
void ordered_data_to_file(global_context_t* ctx)

{

    //MPI_Barrier(ctx -> mpi_communicator);

    MPI_DB_PRINT("[MASTER] writing to file\n");

    float_t* tmp_data; 

    int* ppp; 

    int* displs;
@@ -1630,12 +1636,13 @@ void simulate_master_read_and_scatter(int dims, size_t n, global_context_t *ctx) 


	if (ctx->mpi_rank == 0) 

	{

		//data = read_data_file(ctx, "../norm_data/50_blobs_more_var.npy", MY_TRUE);

		data = read_data_file(ctx, "../norm_data/50_blobs.npy", MY_TRUE);

		data = read_data_file(ctx, "../norm_data/50_blobs_more_var.npy", MY_TRUE);

        ctx->dims = 2;

		//data = read_data_file(ctx, "../norm_data/50_blobs.npy", MY_TRUE);

		// std_g0163178_Me14_091_0000

		// data =

		// read_data_file(ctx,"../norm_data/std_g0163178_Me14_091_0000",MY_TRUE);

		//data = read_data_file(ctx,"../norm_data/std_LR_091_0001",MY_TRUE);

        //ctx->dims = 5;



		// ctx -> n_points = 48*5*2000;

		ctx->n_points = ctx->n_points / ctx->dims;

		//ctx -> n_points = 6 * 500;
@@ -1717,10 +1724,10 @@ void simulate_master_read_and_scatter(int dims, size_t n, global_context_t *ctx) 
	mpi_ngbh_search(ctx, dp_info, &tree, &local_tree, ctx -> local_data, k);

		



    ordered_data_to_file(ctx);

	top_tree_free(ctx, &tree);



	kdtree_v2_free(&local_tree);

    ordered_data_to_file(ctx);

	free(send_counts);

	free(displacements);

	free(dp_info);