Commit aaa52d9d authored by David P. Mayer's avatar David P. Mayer Committed by GitHub
Browse files

Added GXP support to read_gpf() (#164) 

* Added GXP support to read_gpf()
Modified read_gpf() in io_bae.py to support reading of either legacy Socet Set-style GPF files or new Socet GXP-style GPF files. Function now takes a boolean, `gxp` to flag whether function should interpret input file as SSET or GXP style, defaulst to SSET style for backwards compatibility. Removed unused variables from read_gpf(). Field names are now hard-coded in the function, rather than generated dynamically. The hints about field names in the GXP-style GPFs are just complicated enough that hard-coding a list of field names into the function was the simplest solution. Datatypes for each field were already hard-coded, so hard-coding the names is stylistically consistent.

Partially address #24

* Deleted extraneous comment

* Added tests for read_gpf() and updated Socet GXP test data

* Updating test for read_gpf(), ticking minor version number

* Fixing fixtures...maybe

* Fixing fixtures...maybe

* Correct variable assignment in read_gpf(), more fixing fixtures

* Fixing test parameterization

* More test parameterization tweaks
parent 383a1b93

plio/examples/SocetGXP/GXP_example_gpf.csv

0 → 100644
+25 −0
Original line number Diff line number Diff line 
point_id,use,point_type,lat_Y_North,long_X_East,ht,sig0,sig1,sig2,res0,res1,res2,eigenval0,eigenvec0_i,eigenvec0_j,eigenvec0_k,eigenval1,eigenvec1_i,eigenvec1_j,eigenvec1_k,eigenval2,eigenvec2_i,eigenvec2_j,eigenvec2_k

ESP_011969_1425_REDmos_hijitreged_1,1,0,-37.52498315102231,-168.0378101351008,2166.8567751754185,0.0,0.0,0.0,0.0,0.0,0.0,0.010015,0.991258,-0.131935,-0.000733,0.017375,0.126817,0.951241,0.281173,103723.422751,0.036399,0.278808,-0.959657

ESP_011969_1425_REDmos_hijitreged_10,1,0,-37.48836252421589,-168.12422875299504,2312.543966226925,0.0,0.0,0.0,0.0,0.0,0.0,0.009305,0.991449,-0.130497,-0.000525,0.017207,0.125904,0.955474,0.266867,102962.141334,0.034324,0.264651,-0.963733

ESP_011969_1425_REDmos_hijitreged_12,1,0,-37.43168328052415,-168.06327026184894,561.0852310104716,0.0,0.0,0.0,0.0,0.0,0.0,0.009124,0.99128,-0.131771,-0.000682,0.017542,0.126781,0.952304,0.277567,105846.550937,0.035926,0.275233,-0.960706

ESP_011969_1425_REDmos_hijitreged_13,1,0,-37.43513301191926,-168.0950838309726,846.8487925744189,0.0,0.0,0.0,0.0,0.0,0.0,0.008751,0.99136,-0.131167,-0.000616,0.01745,0.126384,0.95393,0.272111,105280.729204,0.035105,0.269838,-0.962266

ESP_011969_1425_REDmos_hijitreged_14,1,0,-37.43831722012688,-168.11377673673576,1129.5974084590293,0.0,0.0,0.0,0.0,0.0,0.0,0.008804,0.991405,-0.130827,-0.000569,0.01738,0.126158,0.954852,0.268963,104835.518938,0.034644,0.266723,-0.96315

ESP_011969_1425_REDmos_hijitreged_15,1,0,-37.43963846641856,-168.1376145220953,1650.4018031283986,0.0,0.0,0.0,0.0,0.0,0.0,0.009152,0.991457,-0.130433,-0.000504,0.017269,0.125897,0.955955,0.265143,104214.901264,0.034102,0.262941,-0.964209

ESP_011969_1425_REDmos_hijitreged_16,1,0,-37.37941962448235,-168.05314041415602,1101.7481752817018,0.0,0.0,0.0,0.0,0.0,0.0,0.009776,0.991225,-0.132182,-0.00073,0.017479,0.127051,0.9512,0.281207,107150.383517,0.036476,0.278832,-0.959647

ESP_011969_1425_REDmos_hijitreged_17,1,0,-37.38083863019242,-168.0675015475748,1181.9292023818236,0.0,0.0,0.0,0.0,0.0,0.0,0.009373,0.991264,-0.131894,-0.000703,0.017444,0.126862,0.951959,0.27871,106929.559928,0.036091,0.276365,-0.960375

ESP_011969_1425_REDmos_hijitreged_18,1,0,-37.3846095508834,-168.10289468543252,1339.499132148808,0.0,0.0,0.0,0.0,0.0,0.0,0.008879,0.991349,-0.13125,-0.000596,0.017364,0.126444,0.953815,0.272486,106414.359157,0.035196,0.270204,-0.96216

ESP_011969_1425_REDmos_hijitreged_19,1,0,-37.38667433736556,-168.12193972900008,1649.1213744038378,0.0,0.0,0.0,0.0,0.0,0.0,0.008908,0.991396,-0.130895,-0.000577,0.01729,0.126212,0.954743,0.269324,105962.787125,0.034702,0.26708,-0.963049

ESP_011969_1425_REDmos_hijitreged_2,1,0,-37.53311540295684,-168.04751417198742,2027.9858571714208,0.0,0.0,0.0,0.0,0.0,0.0,0.009882,0.991283,-0.131745,-0.000716,0.017379,0.126707,0.951862,0.279112,103737.074825,0.03609,0.27677,-0.960258

ESP_011969_1425_REDmos_hijitreged_20,1,0,-37.3877992288042,-168.14528733531682,2249.095866292016,0.0,0.0,0.0,0.0,0.0,0.0,0.009231,0.991447,-0.130512,-0.000505,0.017169,0.125955,0.955802,0.265664,105295.155262,0.03419,0.263455,-0.964066

ESP_011969_1425_REDmos_hijitreged_22,1,0,-37.32506030406861,-168.08145197394458,2133.6427872475433,0.0,0.0,0.0,0.0,0.0,0.0,0.009867,0.991262,-0.131909,-0.000681,0.017285,0.126875,0.951997,0.278574,108618.424031,0.036098,0.276227,-0.960414

ESP_011969_1425_REDmos_hijitreged_23,1,0,-37.3359911666688,-168.10133445338911,2231.769693690932,0.0,0.0,0.0,0.0,0.0,0.0,0.0094,0.991317,-0.131496,-0.00063,0.01724,0.126603,0.953115,0.27485,107858.649273,0.035541,0.272544,-0.961487

ESP_011969_1425_REDmos_hijitreged_24,1,0,-37.34201630935803,-168.12810340862498,2075.5532688013145,0.0,0.0,0.0,0.0,0.0,0.0,0.009299,0.99138,-0.131017,-0.000576,0.017217,0.126313,0.954603,0.269775,107524.900384,0.034796,0.267523,-0.962923

ESP_011969_1425_REDmos_hijitreged_25,1,0,-37.34819057995956,-168.14113429465988,2049.079113123323,0.0,0.0,0.0,0.0,0.0,0.0,0.009376,0.991414,-0.130757,-0.000546,0.017199,0.126146,0.955332,0.26726,107169.466479,0.034425,0.265034,-0.963624

ESP_011969_1425_REDmos_hijitreged_3,1,0,-37.52839666893967,-168.07960079526694,1993.7697023377862,0.0,0.0,0.0,0.0,0.0,0.0,0.009467,0.991364,-0.131138,-0.000638,0.01733,0.126311,0.953542,0.273503,103419.214501,0.035258,0.271221,-0.961871

ESP_011969_1425_REDmos_hijitreged_4,1,0,-37.53132150466094,-168.10855852614645,2020.7308455349735,0.0,0.0,0.0,0.0,0.0,0.0,0.009673,0.991433,-0.130616,-0.00055,0.017278,0.125974,0.955068,0.268284,103104.381955,0.034517,0.266055,-0.96334

ESP_011969_1425_REDmos_hijitreged_5,1,0,-37.53630700111975,-168.1200506995373,2032.7915842058703,0.0,0.0,0.0,0.0,0.0,0.0,0.010626,0.991463,-0.130384,-0.000508,0.019453,0.125817,0.955697,0.266108,102983.822547,0.03421,0.2639,-0.963943

ESP_011969_1425_REDmos_hijitreged_7,1,0,-37.47652424962241,-168.05120054627415,1095.661234358602,0.0,0.0,0.0,0.0,0.0,0.0,0.009388,0.991272,-0.131829,-0.000738,0.017494,0.126796,0.95187,0.279044,104702.232192,0.036083,0.276702,-0.960278

ESP_011969_1425_REDmos_hijitreged_8,1,0,-37.48921839508181,-168.09323595053897,2009.226110327696,0.0,0.0,0.0,0.0,0.0,0.0,0.009011,0.99138,-0.131017,-0.000606,0.017301,0.126237,0.953951,0.272107,103466.816533,0.035073,0.269838,-0.962267

ESP_011969_1425_REDmos_hijitreged_9,1,0,-37.48675126821804,-168.12128226770488,2354.6055057253534,0.0,0.0,0.0,0.0,0.0,0.0,0.009235,0.991445,-0.130527,-0.00054,0.017206,0.125915,0.955302,0.267476,102974.236379,0.034397,0.265255,-0.963564

ESP_047190_1425_REDmos_hijitreged_4,1,0,-37.3298174143407,-168.14113733984277,2007.3591129511703,0.0,0.0,0.0,0.0,0.0,0.0,0.009566,0.991406,-0.130821,-0.000528,0.017203,0.126186,0.955203,0.267701,108124.644303,0.034516,0.265467,-0.963502

ESP_047190_1425_REDmos_hijitreged_5,1,0,-37.33148587629012,-168.15425976288694,1985.726581414384,0.0,0.0,0.0,0.0,0.0,0.0,0.009763,0.991436,-0.130591,-0.000516,0.017183,0.126048,0.955894,0.265288,107981.903787,0.034151,0.263081,-0.964169

plio/examples/SocetGXP/GXP_example_gpf.gpf

+128 −46
Original line number Diff line number Diff line
@@ -18,115 +18,197 @@ GXP_Z_UNITS 3 
GXP_Z_PRECISION     3

GXP_COORD_FORMAT    3

GXP_COORD_FORMAT_PRECISIONS 4

22

point_id,stat,known,lat_Y_North,long_X_East,ht,sig(3),res(3), eigenpair(val, vector)(3)

ESP_020297_2045_REDmos_hijitreged_8bit_1 0 0

24.71012840659569        -18.80031897144174       29.99985045008361       

24

point_id,use,point_type,lat_Y_North,long_X_East,ht,sig(3),res(3), eigenpair(val, vector)(3)

ESP_011969_1425_REDmos_hijitreged_1 1 0

-37.52498315102231       -168.03781013510081      2166.85677517541853     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.010015 0.991258 -0.131935 -0.000733

0.017375 0.126817 0.951241 0.281173

103723.422751 0.036399 0.278808 -0.959657



ESP_020297_2045_REDmos_hijitreged_8bit_2 0 0

24.70421813707974        -18.78227519773831       44.99977570050396       

ESP_011969_1425_REDmos_hijitreged_10 1 0

-37.48836252421589       -168.12422875299504      2312.54396622692502     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009305 0.991449 -0.130497 -0.000525

0.017207 0.125904 0.955474 0.266867

102962.141334 0.034324 0.264651 -0.963733



ESP_020297_2045_REDmos_hijitreged_8bit_6 0 0

24.63946455267877        -18.80461665321617       52.49973861814942       

ESP_011969_1425_REDmos_hijitreged_12 1 0

-37.43168328052415       -168.06327026184894      561.08523101047160      

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009124 0.991280 -0.131771 -0.000682

0.017542 0.126781 0.952304 0.277567

105846.550937 0.035926 0.275233 -0.960706



ESP_020297_2045_REDmos_hijitreged_8bit_7 0 0

24.65291347453150        -18.77556683928612       56.24972002062714       

ESP_011969_1425_REDmos_hijitreged_13 1 0

-37.43513301191926       -168.09508383097261      846.84879257441889      

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.008751 0.991360 -0.131167 -0.000616

0.017450 0.126384 0.953930 0.272111

105280.729204 0.035105 0.269838 -0.962266



ESP_020297_2045_REDmos_hijitreged_8bit_8 0 0

24.64911612548813        -18.73725571127163       58.12471073909546       

ESP_011969_1425_REDmos_hijitreged_14 1 0

-37.43831722012688       -168.11377673673576      1129.59740845902934     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.008804 0.991405 -0.130827 -0.000569

0.017380 0.126158 0.954852 0.268963

104835.518938 0.034644 0.266723 -0.963150



ESP_020297_2045_REDmos_hijitreged_8bit_9 0 0

24.65136761105341        -18.71732124302701       59.06220608808508       

ESP_011969_1425_REDmos_hijitreged_15 1 0

-37.43963846641856       -168.13761452209530      1650.40180312839857     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009152 0.991457 -0.130433 -0.000504

0.017269 0.125897 0.955955 0.265143

104214.901264 0.034102 0.262941 -0.964209



ESP_020297_2045_REDmos_hijitreged_8bit_11 0 0

24.56769120025294        -18.79705461409128       59.53095413767005       

ESP_011969_1425_REDmos_hijitreged_16 1 0

-37.37941962448235       -168.05314041415602      1101.74817528170183     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009776 0.991225 -0.132182 -0.000730

0.017479 0.127051 0.951200 0.281207

107150.383517 0.036476 0.278832 -0.959647



ESP_020297_2045_REDmos_hijitreged_8bit_12 0 0

24.57226969675007        -18.76507418751966       59.76532814686288       

ESP_011969_1425_REDmos_hijitreged_17 1 0

-37.38083863019242       -168.06750154757481      1181.92920238182364     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009373 0.991264 -0.131894 -0.000703

0.017444 0.126862 0.951959 0.278710

106929.559928 0.036091 0.276365 -0.960375



ESP_020297_2045_REDmos_hijitreged_8bit_13 0 0

24.58561888671209        -18.73491604381337       59.88251509185466       

ESP_011969_1425_REDmos_hijitreged_18 1 0

-37.38460955088340       -168.10289468543252      1339.49913214880803     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.008879 0.991349 -0.131250 -0.000596

0.017364 0.126444 0.953815 0.272486

106414.359157 0.035196 0.270204 -0.962160



ESP_020297_2045_REDmos_hijitreged_8bit_14 0 0

24.58274568606691        -18.70979308436567       59.94110856574753       

ESP_011969_1425_REDmos_hijitreged_19 1 0

-37.38667433736556       -168.12193972900008      1649.12137440383776     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.008908 0.991396 -0.130895 -0.000577

0.017290 0.126212 0.954743 0.269324

105962.787125 0.034702 0.267080 -0.963049



ESP_020297_2045_REDmos_hijitreged_8bit_16 0 0

24.49996538364011        -18.77588176749484       59.97040568709735       

ESP_011969_1425_REDmos_hijitreged_2 1 0

-37.53311540295684       -168.04751417198742      2027.98585717142078     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009882 0.991283 -0.131745 -0.000716

0.017379 0.126707 0.951862 0.279112

103737.074825 0.036090 0.276770 -0.960258



ESP_020297_2045_REDmos_hijitreged_8bit_17 0 0

24.50035236811686        -18.76265916231216       59.98505424404698       

ESP_011969_1425_REDmos_hijitreged_20 1 0

-37.38779922880420       -168.14528733531682      2249.09586629201613     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009231 0.991447 -0.130512 -0.000505

0.017169 0.125955 0.955802 0.265664

105295.155262 0.034190 0.263455 -0.964066



ESP_020297_2045_REDmos_hijitreged_8bit_18 0 0

24.50022968395651        -18.72672472647865       59.99237851949499       

ESP_011969_1425_REDmos_hijitreged_22 1 0

-37.32506030406861       -168.08145197394458      2133.64278724754331     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009867 0.991262 -0.131909 -0.000681

0.017285 0.126875 0.951997 0.278574

108618.424031 0.036098 0.276227 -0.960414



ESP_020297_2045_REDmos_hijitreged_8bit_19 0 0

24.50913063816282        -18.69314202812970       59.99604062043176       

ESP_011969_1425_REDmos_hijitreged_23 1 0

-37.33599116666880       -168.10133445338911      2231.76969369093194     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009400 0.991317 -0.131496 -0.000630

0.017240 0.126603 0.953115 0.274850

107858.649273 0.035541 0.272544 -0.961487



ESP_020297_2045_REDmos_hijitreged_8bit_21 0 0

24.42477602090075        -18.77789312511191       59.99787204831861       

ESP_011969_1425_REDmos_hijitreged_24 1 0

-37.34201630935803       -168.12810340862498      2075.55326880131452     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009299 0.991380 -0.131017 -0.000576

0.017217 0.126313 0.954603 0.269775

107524.900384 0.034796 0.267523 -0.962923



ESP_020297_2045_REDmos_hijitreged_8bit_22 0 0

24.42459140840722        -18.75255600552742       59.99878776389185       

ESP_011969_1425_REDmos_hijitreged_25 1 0

-37.34819057995956       -168.14113429465988      2049.07911312332317     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009376 0.991414 -0.130757 -0.000546

0.017199 0.126146 0.955332 0.267260

107169.466479 0.034425 0.265034 -0.963624



ESP_020297_2045_REDmos_hijitreged_8bit_23 0 0

24.44145496656833        -18.72170251844067       59.99924554787116       

ESP_011969_1425_REDmos_hijitreged_3 1 0

-37.52839666893967       -168.07960079526694      1993.76970233778616     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009467 0.991364 -0.131138 -0.000638

0.017330 0.126311 0.953542 0.273503

103419.214501 0.035258 0.271221 -0.961871



ESP_020297_2045_REDmos_hijitreged_8bit_24 0 0

24.43565348607332        -18.70706758301213       59.99947445057103       

ESP_011969_1425_REDmos_hijitreged_4 1 0

-37.53132150466094       -168.10855852614645      2020.73084553497347     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009673 0.991433 -0.130616 -0.000550

0.017278 0.125974 0.955068 0.268284

103104.381955 0.034517 0.266055 -0.963340



ESP_028855_2045_REDmos_hijitreged_8bit_1 0 0

24.69907377638699        -18.75251712669162       59.99958773031716       

ESP_011969_1425_REDmos_hijitreged_5 1 0

-37.53630700111975       -168.12005069953730      2032.79158420587032     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.010626 0.991463 -0.130384 -0.000508

0.019453 0.125817 0.955697 0.266108

102983.822547 0.034210 0.263900 -0.963943



ESP_028855_2045_REDmos_hijitreged_8bit_2 0 0

24.69890626969526        -18.72546009282478       59.99964437624382       

ESP_011969_1425_REDmos_hijitreged_7 1 0

-37.47652424962241       -168.05120054627415      1095.66123435860209     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009388 0.991272 -0.131829 -0.000738

0.017494 0.126796 0.951870 0.279044

104702.232192 0.036083 0.276702 -0.960278



ESP_028855_2045_REDmos_hijitreged_8bit_3 0 0

24.70412810503522        -18.69517420734698       59.99967268034787       

ESP_011969_1425_REDmos_hijitreged_8 1 0

-37.48921839508181       -168.09323595053897      2009.22611032769601     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009011 0.991380 -0.131017 -0.000606

0.017301 0.126237 0.953951 0.272107

103466.816533 0.035073 0.269838 -0.962267



ESP_028855_2045_REDmos_hijitreged_8bit_4 0 0

24.62839745355714        -18.78438969632831       59.99968715603449       

ESP_011969_1425_REDmos_hijitreged_9 1 0

-37.48675126821804       -168.12128226770488      2354.60550572535340     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009235 0.991445 -0.130527 -0.000540

0.017206 0.125915 0.955302 0.267476

102974.236379 0.034397 0.265255 -0.963564



ESP_047190_1425_REDmos_hijitreged_4 1 0

-37.32981741434070       -168.14113733984277      2007.35911295117035     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009566 0.991406 -0.130821 -0.000528

0.017203 0.126186 0.955203 0.267701

108124.644303 0.034516 0.265467 -0.963502



ESP_047190_1425_REDmos_hijitreged_5 1 0

-37.33148587629012       -168.15425976288694      1985.72658141438410     

0.000000 0.000000 0.000000

0.000000 0.000000 0.000000

0.009763 0.991436 -0.130591 -0.000516

0.017183 0.126048 0.955894 0.265288

107981.903787 0.034151 0.263081 -0.964169

plio/examples/SocetGXP/GXP_example_wEigenValues.gpf

deleted100644 → 0
+0 −391

File deleted.

Preview size limit exceeded, changes collapsed.

plio/io/io_bae.py

+46 −37
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ import os 
import json

from collections import defaultdict

from functools import singledispatch



from linecache import getline

import numpy as np

import pandas as pd
@@ -193,61 +193,70 @@ def save_ipf(df, output_path): 
        outIPF.close()

    return



def read_gpf(input_data):

def read_gpf(input_data,gxp=False):

    """

    Read a socet gpf file into a pandas data frame



    Parameters

    ----------

    input_file : str

    input_data : str

                 path to the input data file



    gxp        : bool

                 Flag to notify function if input_data follows Socet GXP format.

                 Defaults is False (assume Socet Set format).



    Returns

    -------

    df : pd.DataFrame

         containing the gpf data with appropriate column names and indices

    """



    # Check that the number of rows is matching the expected number

    with open(input_data, 'r') as f:

        for i, l in enumerate(f):

            if i == 1:

                cnt = int(l)

            elif i == 2:

                col = l

                break



    default_columns = np.genfromtxt(input_data, skip_header=2, dtype='unicode',

                                    max_rows = 1, delimiter = ',')



    columns = []



    for column in default_columns:



        if '(' in column and ')' in column:

            column_name ,suffix = column.split('(')

            num = int(suffix.split(')')[0])



            for column_num in range(int(num)):

                new_column = '{}{}'.format(column_name, column_num)

                columns.append(new_column);



    # line containing point count differs between Socet Set and Socet GXP gpf files

    if not gxp:

        l = 2

    else:

            columns.append(column)

        l = 21



    # Mixed types requires read as unicode - let pandas soft convert

    d = np.genfromtxt(input_data, skip_header=3, dtype='unicode')

    d = d.reshape(-1, 12)



    df = pd.DataFrame(d, columns=columns)

    cnt = getline(input_data, l)

    cnt = int(cnt)

    

    # List of data types for columns in Socet set GPF file

    # Lists of column names and their data types

    if not gxp:

        columns = ['point_id','stat','known','lat_Y_North','long_X_East','ht','sig0','sig1','sig2','res0','res1','res2']

        col_dtype = ['str','int32','int32','float64','float64','float64','float64','float64','float64','float64','float64','float64']

    else:

        columns = ['point_id','use','point_type','lat_Y_North','long_X_East','ht','sig0','sig1','sig2','res0','res1','res2',

                   'eigenval0','eigenvec0_i','eigenvec0_j', 'eigenvec0_k',

                   'eigenval1','eigenvec1_i','eigenvec1_j', 'eigenvec1_k',

                   'eigenval2','eigenvec2_i','eigenvec2_j', 'eigenvec2_k']

        col_dtype = ['str','int32','int32','float64','float64','float64','float64','float64','float64','float64','float64','float64',

                     'float64','float64','float64','float64',

                     'float64','float64','float64','float64',

                     'float64','float64','float64','float64']



    # Build dict of column names and their data types

    dtype_dict = dict(zip(columns, col_dtype))



    # Hard conversion of data types to ensure 'point_id' is treated as string and 'stat' and 'known' flags treated as int

    if not gxp:

        # Mixed types requires read as unicode - let pandas soft convert

        d = np.genfromtxt(input_data, skip_header=(l+1), dtype='unicode')

        d = d.reshape(-1, len(columns))

    else:

        # Read GXP-style GPF a block at a time

        lmax=( (l+1) + ((cnt-1)*8) + 2 )

        for x in range(l+1,lmax,8):

            a = np.genfromtxt(input_data, skip_header=(x), max_rows=4, dtype='unicode')

            b = np.genfromtxt(input_data, skip_header=(x+4), max_rows=3, dtype='unicode')

            if x == (l+1):

                d = np.hstack([np.hstack(a),np.hstack(b)])

            else:

                d = np.vstack(( d, np.hstack([np.hstack(a),np.hstack(b)])) )



    df = pd.DataFrame(d, columns=columns)



    # Hard conversion of data types to ensure 'point_id' is treated as string and 'stat' and 'known' (GXP: 'use', 'point_type')

    #  flags treated as int

    df = df.astype(dtype_dict)



    # Validate the read data with the header point count

plio/io/tests/test_io_bae.py

+50 −13
Original line number Diff line number Diff line 
'''import json

import json

import os



import numpy as np
@@ -14,11 +14,15 @@ import pytest 
def insight_gpf():

    return get_path('InSightE08_XW.gpf')



@pytest.fixture

def gxp_gpf():

    return get_path('GXP_example_gpf.gpf')



@pytest.fixture

def example_str_id_gpf():

    return get_path('InSightE08_string_id.gpf')



@pytest.fixture()

@pytest.fixture

def insight_expected_gpf():

    dtype_dict = {'point_id': 'str',

               'stat': 'int32',
@@ -35,6 +39,34 @@ def insight_expected_gpf(): 
    return pd.read_csv(get_path('InSightE08_XW.csv'), dtype=dtype_dict)



@pytest.fixture

def gxp_expected_gpf():

    dtype_dict = {'point_id': 'str',

               'use': 'int32',

               'point_type': 'int32',

               'lat_Y_North': 'float64',

               'long_X_East': 'float64',

               'ht': 'float64',

               'sig0': 'float64',

               'sig1': 'float64',

               'sig2': 'float64',

               'res0': 'float64',

               'res1': 'float64',

               'res2': 'float64',

               'eigenval0': 'float64',

               'eigenvec0_i': 'float64',

               'eigenvec0_j': 'float64',

               'eigenvec0_k': 'float64',

               'eigenval1': 'float64',

               'eigenvec1_i': 'float64',

               'eigenvec1_j': 'float64',

               'eigenvec1_k': 'float64',

               'eigenval2': 'float64',

               'eigenvec2_i': 'float64',

               'eigenvec2_j': 'float64',

               'eigenvec2_k': 'float64'}

    return pd.read_csv(get_path('GXP_example_gpf.csv'), dtype=dtype_dict)



'''@pytest.fixture

def insight_ipf():

    return get_path('P20_008845_1894_XN_09N203W.ipf')
@@ -62,13 +94,18 @@ def insight_expected_ipf(): 
def test_read_ipf(ipf, expected):

    df = read_ipf(ipf)

    assert_frame_equal(df, expected)

'''

# @pytest.mark.parametrize('gpf, expected', [ (get_path('InSightE08_XW.gpf'),insight_expected_gpf() ) ] )

def test_read_gpf(insight_gpf, insight_expected_gpf):

    df = read_gpf(insight_gpf)

    assert_frame_equal(df, insight_expected_gpf)



@pytest.mark.parametrize('gpf, expected', [(insight_gpf(),insight_expected_gpf())])

def test_read_gpf(gpf, expected):

    df = read_gpf(gpf)

    assert_frame_equal(df, expected)

# @pytest.mark.parametrize('gpf, expected', [ (get_path('GXP_example_gpf.gpf'),gxp_expected_gpf() ) ] )

def test_read_gxp_gpf(gxp_gpf, gxp_expected_gpf):

    df = read_gpf(gxp_gpf,gxp=True)

    assert_frame_equal(df, gxp_expected_gpf)



@pytest.mark.parametrize('ipf, file', [(insight_ipf(), 'plio/io/tests/temp')])

'''@pytest.mark.parametrize('ipf, file', [(insight_ipf(), 'plio/io/tests/temp')])

def test_write_ipf(ipf, file):

    df = read_ipf(ipf)

    save_ipf(df, file)
@@ -163,8 +200,8 @@ def test_write_str_id_gpf(gpf, file): 
    # Test that every 5th line (the lines containing the point ID and integer flags) matches

    for i in range(3,len(fs),5):

        assert fs[i] == fl[i]



@pytest.mark.parametrize('gpf', [(example_str_id_gpf())])

'''

@pytest.mark.parametrize('gpf', [get_path('InSightE08_string_id.gpf')])

def test_gpf_dtypes(gpf):

    """

    This test makes sure that a GPF whose point IDs only contain numbers
@@ -182,7 +219,7 @@ def test_gpf_dtypes(gpf): 
    # Check that the type of each column matches the truth list

    assert truth_dtypes == test_dtypes



@pytest.mark.parametrize('ipf', [(example_str_id_ipf())])

'''@pytest.mark.parametrize('ipf', [(example_str_id_ipf())])

def test_ipf_dtypes(ipf):

    """

    This test makes sure that a IPF whose point IDs only contain numbers