B C[CaQ@s"ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl ZddlmZddlmZddlmZddlmZddlmZmZddlZddlZddlZ ddl!m"Z"ddl!m!Z!dd l!m#Z#dd l!m$Z$d d Z%d dZ&ddZ'ddZ(ddZ)ddZ*dddZ+dS)N) interpolate)Time)PdfPages)pyplot)convolve Box1DKernel)calc_targOlapf)calc_shk) old_calc_shk)old_hk_windowsc Cs2t||t||d}t|}WdQRX|S)Nrb)create_flat_dict_fileopenpickleload)flatPath flatPicklefflatDictr0/home/travis/obs/LCO/nreshk/NRES_SHK_Pipeline.pyNRES_SHK_MkFlat-s rc Cst}|dti|dti|dti|dtixt|D]\}}}x||D]t}|drbtjjtj ||}|dj }t |d} |d} |d } | tj ||f} || } | | | iqbWqRWt||d }t ||WdQRXdS) Ncpttlvlscelpz.fitsrzMJD-OBSSITEIDNAXIS2wb)dictupdateoswalkendswithastropyiofitsrpathjoinheaderfloatrdump)ZflatDirfileNamerr'subdirsfilesnameZflHDu1r)mjdsitenOrdtuplZsiteDictrrrrr ;s$    r c Csd}tj|}|dd|d}|dd|d}|}tjj|}tjj|}|djd}|dj} t |dkrt |dtjjj j j kry|djdd} Wq|djdd} YqXn*t |dkr|djd} n|djd} t| || |d}|||S)Nrz-noflatz-waveZ WavelenStarZ WavelengthT)r!r'splitextr$r%r&rdatar)lentypehdutable BinTableHDUhrawDataclose) obsFileNamer6spltnoFlatwaver,ZspecHDuZwvHDuspecr)waveGridrrrold_multifile_NRES_to_data]s*  & rEc Csg}g}g}d}d}d}tj|}|dd|d}tj|rHd}|rTt|Stjj|} | dj }| dj }| dj }t |||||}| |S)NFrz-noflatr4T)r!r'r5existsrEr$r%r&rr6r)r<r=r>) r?rDrCr)r,Z oldFormatr6ZsplitFrAZoHDurrrload_obs_for_pipelines&     rIc Csj|j}|j}ttj|}ddg}d}t|}|j}g} xt|D]} | ddgqBWd} |d|d|d} |d|tj| tj d } tj ||ftj d }tj ||ftj d }xftt|D]V}t ||ddf||ddf<||ddf||t |d f||ddf<qW|d|ddf}tj ||ftj d }g}xtt|D]} d|| d| | df<d|| | | ddf<ttt|| ddf|| ddftd td td || ddf<qXWtj t | tj d }xPtt|D]@} tj||| ddf|| ddfd d d}||| }qW|jdd}t|dkr`d} | || fS)Ngw@g@z@gMbP?iXiFr4r)dtypelinear extrapolate)kind fill_value)ming?T)rDr2nparanger<lowGOrdr7nxrangeappendfloat64zerosgradientintrrrinterp1dclipmax)rawflatlamhighOrdgOrdZlamRandLamZnGordrUboundsibadnLamlamGridZdLambxscaleyZgFlatZsgFlatstuffflatOlap interpfuncrrr mk_flatolapsB"4P . rocCsdS)Nr)obsrrrfind_and_make_flatsrqc 5sd} d} t|} i} x| D]} g}g}g}g}g}t| sTtd| dq|dk r| |krr|dkrd}qt| dqntj|| rt| dqtd td | td | } tj|| }tj|d d d }dd|D}tdt t |x |D]}t |dkr&q |dk rFt j|krFq j|krVq td| _|j}j}fdd|D}t|tj}||d}tjjtj||}|dj}||dk rjdkrq |dkrjdkrd_t|}|d}|d} t| |}!tj|dd|d| |!d| d}"t| |!|"}#t |dd|d| |!| }$|$d}%y`|d!j}|d|}&tdt jx,t"t |&D]}'|&dk r|&d|"|'<qWWn YnXt#| |!|#}(|(d})|(d}*d}+d},t$| |!}-|-tj%krJd }+|&j|,d7},n&|dd krp|&jd }+|,d7},t'| ||!|)|"d|+|#| }.|.(}/|.j)}0t|.*d t |-tj+dkrt,-| |%|.d!|+rtt j|,q |.jdkr|&|.n |&|.q Wt.|| |%}t.|| |%}||}t/|.(ggg}1x.|D]&}2|1d&|2j|1d&|2j0qRWg}3|dkrt|.(| d"d#}4t12|4}3WdQRX|dkrt|.(| d$d%}4t13||4WdQRXt|.(| d&d%}4t13||4WdQRX|dkrt|.(| d$d%}4t13|3|4WdQRXt|.(| d&d%}4t13||4WdQRX||3}t|.(| d"d%}4t13||4WdQRXt|t,4||| 5| |iqW|S)'aMain NRES SHK Function with various options to create time-series for input stars. dataPath- location of data folder outputPath- location of output folder flatDict- dictionary of flat files created with create_flat_dict_file lab- reference spectra to align against skip- List of MJD's to skip due to bad data or testing forceRun- stars to run even if they have an output folder already manualAdj- list of MJD's coupled with 4 offsets(analyzedData.offset) to manually adjust bad shifting only- If it exists, the pipeline will ignore observations with MJD's not in this list new68- 9/19/2020 Temporary variable to test 67/68 order differences rFg{Gz?z bad folder ''Nrz is not in force runz already has outputzN++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++zRunning pipeline on HD z//**//*-e91.fitsT) recursivecSsg|]}d|kr|qS)outputr).0rrrr jsz%NRES_SHK_Pipeline..zNumber of observations: zO-------------------------------------------------------------------------------cs$i|]\}}|djkr||qS)r)r2)rukeyvalue)obsRawrr sz%NRES_SHK_Pipeline..r4CD rL)rjsmoothzmanual offset for mjd Fz Bad: z bad spec detector.rKz bad flat. z with bad ratio: g333333?z.pklr z_O67.pklrz_N68.pkl)6r<get_immediate_subdirectoriesis_folder_starprintr!r'rHr(globstrr7rIrRfloorr0starr1r2items closestKeyr*r$r%r&rr6r>rorpipefind_window_offsetscreate_integration_windows calc_del_lamindexrVr bad_spec_detection_v2 maxNoiseRatiorW analyzedDatastarDir decimalYrlabelpdfModeplotpdf_from_intermediate_datasum_daily_datamkdir_pshkrrr+plot_timeseriesr )5dataPath outputPathrlabskipZforceRunZ manualAdjonlyZnew68starNameresstarsZstarDatasZobsFilesZ analyzed_67Z analyzed_68ZanalyzedZbadDrfZstarPathZobsUnr?ZfDr2ZflatSubZfKZ flatFilePathZfHDur`ZflatRetrmri targOlapfoffsetsZintegrationWinsZlaboutlabSpecZoffsoshkRetrwindowsZbadSpecZ badReasonZ noiseRatiooData outputDirrtdoldrr)rzrNRES_SHK_Pipeline0s                  "                r)NN),sysr!shutilrloggingrpprintastropy.io.fitsr$timeZscipy.constants constantsscnumpyrRscipyr astropy.timermatplotlib.backends.backend_pdfr matplotlibrpltastropy.convolutionrrhelpersr<pipelinerplottingrr rr r rr rErIrorqrrrrrs:         ",+d