Commit 0894606b authored by Nandhana Sakhtivel's avatar Nandhana Sakhtivel
Browse files

Restructing of i/o

parent f61fbf14

Makefile

+7 −7
Original line number Diff line number Diff line
@@ -39,8 +39,8 @@ OPT += -DWRITE_IMAGE 
OPT += -DPHASE_ON





DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c 

COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o

DEPS = w-stacking.h main.c w-stacking.cu phase_correction.cu allvars.h init.c gridding.c fourier_transform.c result.c

COBJ = w-stacking.o main.o phase_correction.o allvars.o init.o gridding.o fourier_transform.o result.o



w-stacking.c: w-stacking.cu

	cp w-stacking.cu w-stacking.c
@@ -60,17 +60,17 @@ serial: $(COBJ) 
	$(CC) $(OPTIMIZE) $(OPT) -o w-stackingCfftw_serial  $^ $(LIBS)



serial_omp: phase_correction.c

	$(CC)  $(OPTIMIZE) $(OPT) -o w-stackingOMP_serial main.c init.c w-stacking_omp.c    $(CFLAGS) $(LIBS)

	$(CC)  $(OPTIMIZE) $(OPT) -o w-stackingOMP_serial main.c init.c gridding.c fourier_transform.c result.c w-stacking_omp.c    $(CFLAGS) $(LIBS)



simple_mpi: phase_correction.c

	$(MPICC) $(OPTIMIZE) $(OPT) -o w-stackingMPI_simple w-stacking_omp.c main.c init.c phase_correction.c  $(CFLAGS) $(LIBS)

	$(MPICC) $(OPTIMIZE) $(OPT) -o w-stackingMPI_simple w-stacking_omp.c main.c init.c gridding.c fourier_transform.c result.c phase_correction.c  $(CFLAGS) $(LIBS)



mpi_omp: phase_correction.c

	$(MPICC) $(OPTIMIZE) $(OPT) -o w-stackingMPI_omp w-stacking_omp.c main.c init.c phase_correction.c  $(CFLAGS) $(LIBS)

	$(MPICC) $(OPTIMIZE) $(OPT) -o w-stackingMPI_omp w-stacking_omp.c main.c init.c gridding.c fourier_transform.c result.c phase_correction.c  $(CFLAGS) $(LIBS)



serial_cuda:

	$(NVCC) $(NVFLAGS) -c w-stacking.cu phase_correction.cu $(NVLIB)

	$(CC)  $(OPTIMIZE) $(OPT) -c main.c init.c $(CFLAGS) $(LIBS)

	$(CC)  $(OPTIMIZE) $(OPT) -c main.c init.c gridding.c fourier_transform.c result.c $(CFLAGS) $(LIBS)

	$(CXX) $(OPTIMIZE) $(OPT) -o w-stackingfftw_serial w-stacking-fftw.o w-stacking.o phase_correction.o $(CFLAGS) $(NVLIB) -lm



mpi: $(COBJ)
@@ -78,7 +78,7 @@ mpi: $(COBJ) 


mpi_cuda:

	$(NVCC)   $(NVFLAGS) -c w-stacking.cu phase_correction.cu $(NVLIB)

	$(MPICC)  $(OPTIMIZE) $(OPT) -c main.c init.c$(CFLAGS) $(LIBS)

	$(MPICC)  $(OPTIMIZE) $(OPT) -c main.c init.c fourier_transform.c result.c $(CFLAGS) $(LIBS)

	$(MPIC++) $(OPTIMIZE) $(OPT)   -o w-stackingfftw w-stacking-fftw.o w-stacking.o phase_correction.o $(NVLIB) $(CFLAGS) $(LIBS)



clean:

allvars.c

+14 −3
Original line number Diff line number Diff line
@@ -4,14 +4,16 @@ struct io file; 


struct ip in;



struct op out = {"grid.txt", "coords.txt", "grid_real.bin", "grid_img.bin", "fft.txt", "fft_real.bin", "fft_img.bin", "run.log", ".txt", "timings.dat"};

struct op out;

struct op outparam = {"grid.txt", "coords.txt", "grid_real.bin", "grid_img.bin", "fft.txt", "fft_real.bin", "fft_img.bin", "run.log", ".txt", "timings.dat"};



struct meta metaData;

struct time timing;

struct parameter param;

struct fileData data;



char filename[1000];

int w_support = 7;

char filename[1000], buf[30], num_buf[30];

char datapath[900];

int xaxis, yaxis;

int grid_size_x = 2048;
@@ -21,7 +23,16 @@ int rank; 
int size;

long nsectors;

long startrow;

double resolution;

double resolution, dx, dw, w_supporth;



clock_t start, end, start0, startk, endk;

struct timespec begin, finish, begin0, begink, finishk;



struct sectorlist ** sectorhead;

long * histo_send;

double * grid, *gridss;

fftw_complex *fftwgrid;



#ifdef USE_MPI

    MPI_Win slabwin;

#endif

allvars.h

+17 −4
Original line number Diff line number Diff line
@@ -61,7 +61,7 @@ extern struct op 
        char extension[30];

        char timingfile[30];



} out;

} out, outparam;



extern struct meta

{
@@ -108,17 +108,30 @@ extern struct fileData 
        float * visimg;

}data;



extern char filename[1000];

extern struct sectorlist {

     long index;

     struct sectorlist * next;

}** sectorhead;





extern char filename[1000], buf[30], num_buf[30];

extern char datapath[900];

extern int xaxis, yaxis;

extern int grid_size_x;

extern int grid_size_y;

extern int num_w_planes;

extern int num_w_planes, w_support;

extern int rank;

extern int size;

extern long nsectors;

extern long startrow;

extern double resolution;

extern double resolution, dx, dw, w_supporth;



extern clock_t start, end, start0, startk, endk;

extern struct timespec begin, finish, begin0, begink, finishk;

extern long * histo_send;

extern double * grid, *gridss;

extern fftw_complex *fftwgrid;



#ifdef USE_MPI

    extern  MPI_Win slabwin;

#endif

data/paramfile.txt

0 → 100644
+13 −0
Original line number Diff line number Diff line 
ndatasets              3

Datapath1              /u/nsakthivel/LOFAR/data/newgauss2noconj_t201806301100_SBL180.binMS/ 

Datapath2              /u/nsakthivel/LOFAR/data/newgauss2noconj_t201806301100_SBL180.binMS/

Datapath3              /u/nsakthivel/LOFAR/data/newgauss2noconj_t201806301100_SBL180.binMS/

num_threads            2

ufile		       ucoord.bin

vfile    	       vcoord.bin

wfile                  wcoord.bin

weightsfile	       weights.bin

visrealfile            visibilities_real.bin

visimgfile             visibilities_img.bin

metafile               meta.txt

fourier_transform.c

0 → 100644
+213 −0
Original line number Diff line number Diff line 
#include <stdio.h>

#include "allvars.h"

#include "proto.h"



void fftw_data(){



    #ifdef USE_FFTW

        // FFT transform the data (using distributed FFTW)

        if(rank == 0)printf("PERFORMING FFT\n");

        clock_gettime(CLOCK_MONOTONIC, &begin);

        start = clock();

        fftw_plan plan;

        ptrdiff_t alloc_local, local_n0, local_0_start;

        double norm = 1.0/(double)(grid_size_x*grid_size_y);



        // map the 1D array of complex visibilities to a 2D array required by FFTW (complex[*][2])

        // x is the direction of contiguous data and maps to the second parameter

        // y is the parallelized direction and corresponds to the first parameter (--> n0)

        // and perform the FFT per w plane

        alloc_local = fftw_mpi_local_size_2d(grid_size_y, grid_size_x, MPI_COMM_WORLD,&local_n0, &local_0_start);

        fftwgrid = fftw_alloc_complex(alloc_local);

        plan = fftw_mpi_plan_dft_2d(grid_size_y, grid_size_x, fftwgrid, fftwgrid, MPI_COMM_WORLD, FFTW_BACKWARD, FFTW_ESTIMATE);



        long fftwindex = 0;

        long fftwindex2D = 0;

        for (int iw=0; iw<num_w_planes; iw++)

        {

            //printf("FFTing plan %d\n",iw);

            //select the w-plane to transform

            for (int iv=0; iv<yaxis; iv++)

            {

               for (int iu=0; iu<xaxis; iu++)

               {

                   fftwindex2D = iu + iv*xaxis;

                   fftwindex = 2*(fftwindex2D + iw*xaxis*yaxis);

                   fftwgrid[fftwindex2D][0] = grid[fftwindex];

                   fftwgrid[fftwindex2D][1] = grid[fftwindex+1];

               }

            }



            // do the transform for each w-plane        

            fftw_execute(plan);



            // save the transformed w-plane

            for (int iv=0; iv<yaxis; iv++)

            {

               for (int iu=0; iu<xaxis; iu++)

               {

                   fftwindex2D = iu + iv*xaxis;

                   fftwindex = 2*(fftwindex2D + iw*xaxis*yaxis);

                   gridss[fftwindex] = norm*fftwgrid[fftwindex2D][0];

                   gridss[fftwindex+1] = norm*fftwgrid[fftwindex2D][1];

               }

            }



        }



        fftw_destroy_plan(plan);



        #ifdef USE_MPI

        MPI_Win_fence(0,slabwin);

        MPI_Barrier(MPI_COMM_WORLD);

        #endif



        end = clock();

        clock_gettime(CLOCK_MONOTONIC, &finish);

        timing.fftw_time = ((double) (end - start)) / CLOCKS_PER_SEC;

        timing.fftw_time1 = (finish.tv_sec - begin.tv_sec);

        timing.fftw_time1 += (finish.tv_nsec - begin.tv_nsec) / 1000000000.0;

        clock_gettime(CLOCK_MONOTONIC, &begin);



    #endif

}



void write_fftw_data(){



   #ifdef USE_FFTW

     #ifdef WRITE_DATA

        // Write results

        #ifdef USE_MPI

        MPI_Win writewin;

        MPI_Win_create(gridss, size_of_grid*sizeof(double), sizeof(double), MPI_INFO_NULL, MPI_COMM_WORLD, &writewin);

        MPI_Win_fence(0,writewin);

        #endif

        if (rank == 0)

        {

          printf("WRITING FFT TRANSFORMED DATA\n");

          file.pFilereal = fopen (out.fftfile2,"wb");

          file.pFileimg = fopen (out.fftfile3,"wb");

          #ifdef USE_MPI

          for (int isector=0; isector<nsectors; isector++)

          {

              MPI_Win_lock(MPI_LOCK_SHARED,isector,0,writewin);

              MPI_Get(gridss_w,size_of_grid,MPI_DOUBLE,isector,0,size_of_grid,MPI_DOUBLE,writewin);

              MPI_Win_unlock(isector,writewin);

              for (long i=0; i<size_of_grid/2; i++)

              {

                      gridss_real[i] = gridss_w[2*i];

                      gridss_img[i] = gridss_w[2*i+1];

              }

              if (num_w_planes > 1)

              {

                for (int iw=0; iw<num_w_planes; iw++)

                for (int iv=0; iv<yaxis; iv++)

                for (int iu=0; iu<xaxis; iu++)

                {

                          long global_index = (iu + (iv+isector*yaxis)*xaxis + iw*grid_size_x*grid_size_y)*sizeof(double);

                          long index = iu + iv*xaxis + iw*xaxis*yaxis;

                          fseek(file.pFilereal, global_index, SEEK_SET);

                          fwrite(&gridss_real[index], 1, sizeof(double), file.pFilereal);

                }

                for (int iw=0; iw<num_w_planes; iw++)

                for (int iv=0; iv<yaxis; iv++)

                for (int iu=0; iu<xaxis; iu++)

                {

                          long global_index = (iu + (iv+isector*yaxis)*xaxis + iw*grid_size_x*grid_size_y)*sizeof(double);

                          long index = iu + iv*xaxis + iw*xaxis*yaxis;

                          fseek(file.pFileimg, global_index, SEEK_SET);

                          fwrite(&gridss_img[index], 1, sizeof(double), file.pFileimg);

                }

               } 

               else 

               {

                          fwrite(gridss_real, size_of_grid/2, sizeof(double), file.pFilereal);

                          fwrite(gridss_img, size_of_grid/2, sizeof(double), file.pFileimg);

               }



           }

           #else

           /*

               for (int iw=0; iw<num_w_planes; iw++)

                 for (int iv=0; iv<grid_size_y; iv++)

                   for (int iu=0; iu<grid_size_x; iu++)

                   {

                          int isector = 0;

                          long index = 2*(iu + iv*grid_size_x + iw*grid_size_x*grid_size_y);

                          double v_norm = sqrt(gridtot[index]*gridtot[index]+gridtot[index+1]*gridtot[index+1]);

                          fprintf (file.pFile, "%d %d %d %f %f %f\n", iu,iv,iw,gridtot[index],gridtot[index+1],v_norm);

                   }

           */

           #endif



           fclose(file.pFilereal);

           fclose(file.pFileimg);

        }

        #ifdef USE_MPI

               MPI_Win_fence(0,writewin);

               MPI_Win_free(&writewin);

               MPI_Barrier(MPI_COMM_WORLD);

        #endif

     #endif //WRITE_DATA



     fftw_free(fftwgrid);



     // Phase correction  



     clock_gettime(CLOCK_MONOTONIC, &begin);

     start = clock();

     if(rank == 0)printf("PHASE CORRECTION\n");

     double* image_real = (double*) calloc(xaxis*yaxis,sizeof(double));

     double* image_imag = (double*) calloc(xaxis*yaxis,sizeof(double));



     phase_correction(gridss,image_real,image_imag,xaxis,yaxis,num_w_planes,grid_size_x,grid_size_y,resolution,metaData.wmin,metaData.wmax,param.num_threads);



     end = clock();

     clock_gettime(CLOCK_MONOTONIC, &finish);

     timing.phase_time = ((double) (end - start)) / CLOCKS_PER_SEC;

     timing.phase_time1 = (finish.tv_sec - begin.tv_sec);

     timing.phase_time1 += (finish.tv_nsec - begin.tv_nsec) / 1000000000.0;



     #ifdef WRITE_IMAGE



        if(rank == 0)

        {

            file.pFilereal = fopen (out.fftfile2,"wb");

            file.pFileimg = fopen (out.fftfile3,"wb");

            fclose(file.pFilereal);

            fclose(file.pFileimg);

        }

        #ifdef USE_MPI

            MPI_Barrier(MPI_COMM_WORLD);

        #endif

        if(rank == 0)printf("WRITING IMAGE\n");

        for (int isector=0; isector<size; isector++)

        {

            #ifdef USE_MPI

                MPI_Barrier(MPI_COMM_WORLD);

            #endif

            if(isector == rank)

            {

               printf("%d writing\n",isector);

               file.pFilereal = fopen (out.fftfile2,"ab");

               file.pFileimg = fopen (out.fftfile3,"ab");



               long global_index = isector*(xaxis*yaxis)*sizeof(double);



               fseek(file.pFilereal, global_index, SEEK_SET);

               fwrite(image_real, xaxis*yaxis, sizeof(double), file.pFilereal);

               fseek(file.pFileimg, global_index, SEEK_SET);

               fwrite(image_imag, xaxis*yaxis, sizeof(double), file.pFileimg);



               fclose(file.pFilereal);

               fclose(file.pFileimg);

            }

        }

        #ifdef USE_MPI

           MPI_Barrier(MPI_COMM_WORLD);

        #endif



     #endif //WRITE_IMAGE



  #endif  //FFTW

}