52class open():
 53    """
 54    GRIB2 File Object.  A physical file can contain one or more GRIB2 messages.  When instantiated,
 55    class `grib2io.open`, the file named `filename` is opened for reading (`mode = 'r'`) and is
 56    automatically indexed.  The indexing procedure reads some of the GRIB2 metadata for all GRIB2 Messages.
 57
 58    A GRIB2 Message may contain submessages whereby Section 2-7 can be repeated.  grib2io accommodates
 59    for this by flattening any GRIB2 submessages into multiple individual messages.
 60
 61    Attributes
 62    ----------
 63
 64    **`mode`** File IO mode of opening the file.
 65
 66    **`name`** Full path name of the GRIB2 file.
 67
 68    **`messages`** Count of GRIB2 Messages contained in the file.
 69
 70    **`current_message`** Current position of the file in units of GRIB2 Messages.
 71
 72    **`size`** Size of the file in units of bytes.
 73
 74    **`closed`** `True` is file handle is close; `False` otherwise.
 75
 76    **`variables`** Tuple containing a unique list of variable short names (i.e. GRIB2 abbreviation names).
 77
 78    **`levels`** Tuple containing a unique list of wgrib2-formatted level/layer strings.
 79    """
 80    __slots__ = ('_filehandle','_hasindex','_index','mode','name','messages',
 81                 'current_message','size','closed','variables','levels','_pos')
 82    def __init__(self, filename, mode='r', **kwargs):
 83        """
 84        `open` Constructor
 85
 86        Parameters
 87        ----------
 88
 89        **`filename : str`**
 90
 91        File name containing GRIB2 messages.
 92
 93        **`mode : str, optional`**
 94
 95        File access mode where `r` opens the files for reading only; `w` opens the file for writing.
 96        """
 97        if mode in {'a','r','w'}:
 98            mode = mode+'b'
 99            if 'w' in mode: mode += '+'
100        self._filehandle = builtins.open(filename,mode=mode,buffering=ONE_MB)
101        self._hasindex = False
102        self._index = {}
103        self.mode = mode
104        self.name = os.path.abspath(filename)
105        self.messages = 0
106        self.current_message = 0
107        self.size = os.path.getsize(self.name)
108        self.closed = self._filehandle.closed
109        self.levels = None
110        self.variables = None
111        if 'r' in self.mode:
112            try:
113                self._build_index(no_data=kwargs['_xarray_backend'])
114            except(KeyError):
115                self._build_index()
116        # FIX: Cannot perform reads on mode='a'
117        #if 'a' in self.mode and self.size > 0: self._build_index()
118
119
120    def __delete__(self, instance):
121        """
122        """
123        self.close()
124        del self._index
125
126
127    def __enter__(self):
128        """
129        """
130        return self
131
132
133    def __exit__(self, atype, value, traceback):
134        """
135        """
136        self.close()
137
138
139    def __iter__(self):
140        """
141        """
142        yield from self._index['msg']
143
144
145    def __len__(self):
146        """
147        """
148        return self.messages
149
150
151    def __repr__(self):
152        """
153        """
154        strings = []
155        for k in self.__slots__:
156            if k.startswith('_'): continue
157            strings.append('%s = %s\n'%(k,eval('self.'+k)))
158        return ''.join(strings)
159
160
161    def __getitem__(self, key):
162        """
163        """
164        if isinstance(key,int):
165            if abs(key) >= len(self._index['msg']):
166                raise IndexError("index out of range")
167            else:
168                return self._index['msg'][key]
169        elif isinstance(key,str):
170            return self.select(shortName=key)
171        elif isinstance(key,slice):
172            return self._index['msg'][key]
173        else:
174            raise KeyError('Key must be an integer, slice, or GRIB2 variable shortName.')
175
176
177    def _build_index(self, no_data=False):
178        """
179        Perform indexing of GRIB2 Messages.
180        """
181        # Initialize index dictionary
182        if not self._hasindex:
183            self._index['offset'] = []
184            self._index['bitmap_offset'] = []
185            self._index['data_offset'] = []
186            self._index['size'] = []
187            self._index['data_size'] = []
188            self._index['submessageOffset'] = []
189            self._index['submessageBeginSection'] = []
190            self._index['isSubmessage'] = []
191            self._index['messageNumber'] = []
192            self._index['msg'] = []
193            self._hasindex = True
194
195        # Iterate
196        while True:
197            try:
198                # Read first 4 bytes and decode...looking for "GRIB"
199                pos = self._filehandle.tell()
200                header = struct.unpack('>i',self._filehandle.read(4))[0]
201
202                # Test header. Then get information from GRIB2 Section 0: the discipline
203                # number, edition number (should always be 2), and GRIB2 message size.
204                # Then iterate to check for submessages.
205                if header.to_bytes(4,'big') == b'GRIB':
206
207                    _issubmessage = False
208                    _submsgoffset = 0
209                    _submsgbegin = 0
210                    _bmapflag = None
211
212                    # Read the rest of Section 0 using struct.
213                    section0 = np.concatenate(([header],list(struct.unpack('>HBBQ',self._filehandle.read(12)))),dtype=np.int64)
214                    assert section0[3] == 2
215
216                    # Read and unpack Section 1
217                    secsize = struct.unpack('>i',self._filehandle.read(4))[0]
218                    secnum = struct.unpack('>B',self._filehandle.read(1))[0]
219                    assert secnum == 1
220                    self._filehandle.seek(self._filehandle.tell()-5)
221                    _grbmsg = self._filehandle.read(secsize)
222                    _grbpos = 0
223                    section1,_grbpos = g2clib.unpack1(_grbmsg,_grbpos,np.empty)
224                    secrange = range(2,8)
225                    while 1:
226                        section2 = b''
227                        for num in secrange:
228                            secsize = struct.unpack('>i',self._filehandle.read(4))[0]
229                            secnum = struct.unpack('>B',self._filehandle.read(1))[0]
230                            if secnum == num:
231                                if secnum == 2:
232                                    if secsize > 0:
233                                        section2 = self._filehandle.read(secsize-5)
234                                elif secnum == 3:
235                                    self._filehandle.seek(self._filehandle.tell()-5)
236                                    _grbmsg = self._filehandle.read(secsize)
237                                    _grbpos = 0
238                                    # Unpack Section 3
239                                    _gds,_gdt,_deflist,_grbpos = g2clib.unpack3(_grbmsg,_grbpos,np.empty)
240                                    _gds = _gds.tolist()
241                                    _gdt = _gdt.tolist()
242                                    section3 = np.concatenate((_gds,_gdt))
243                                    section3 = np.where(section3==4294967295,-1,section3)
244                                elif secnum == 4:
245                                    self._filehandle.seek(self._filehandle.tell()-5)
246                                    _grbmsg = self._filehandle.read(secsize)
247                                    _grbpos = 0
248                                    # Unpack Section 4
249                                    _numcoord,_pdt,_pdtnum,_coordlist,_grbpos = g2clib.unpack4(_grbmsg,_grbpos,np.empty)
250                                    _pdt = _pdt.tolist()
251                                    section4 = np.concatenate((np.array((_numcoord,_pdtnum)),_pdt))
252                                elif secnum == 5:
253                                    self._filehandle.seek(self._filehandle.tell()-5)
254                                    _grbmsg = self._filehandle.read(secsize)
255                                    _grbpos = 0
256                                    # Unpack Section 5
257                                    _drt,_drtn,_npts,self._pos = g2clib.unpack5(_grbmsg,_grbpos,np.empty)
258                                    section5 = np.concatenate((np.array((_npts,_drtn)),_drt))
259                                    section5 = np.where(section5==4294967295,-1,section5)
260                                elif secnum == 6:
261                                    # Unpack Section 6. Not really...just get the flag value.
262                                    _bmapflag = struct.unpack('>B',self._filehandle.read(1))[0]
263                                    if _bmapflag == 0:
264                                        _bmappos = self._filehandle.tell()-6
265                                    elif _bmapflag == 254:
266                                        pass # Do this to keep the previous position value
267                                    else:
268                                        _bmappos = None
269                                    self._filehandle.seek(self._filehandle.tell()+secsize-6)
270                                elif secnum == 7:
271                                    # Unpack Section 7. No need to read it, just index the position in file.
272                                    _datapos = self._filehandle.tell()-5
273                                    _datasize = secsize
274                                    self._filehandle.seek(self._filehandle.tell()+secsize-5)
275                                else:
276                                    self._filehandle.seek(self._filehandle.tell()+secsize-5)
277                            else:
278                                if num == 2 and secnum == 3:
279                                    pass # Allow this.  Just means no Local Use Section.
280                                else:
281                                    _issubmessage = True
282                                    _submsgoffset = (self._filehandle.tell()-5)-(self._index['offset'][-1])
283                                    _submsgbegin = secnum
284                                self._filehandle.seek(self._filehandle.tell()-5)
285                                continue
286                        trailer = struct.unpack('>4s',self._filehandle.read(4))[0]
287                        if trailer == b'7777':
288                            self.messages += 1
289                            self._index['offset'].append(pos)
290                            self._index['bitmap_offset'].append(_bmappos)
291                            self._index['data_offset'].append(_datapos)
292                            self._index['size'].append(section0[-1])
293                            self._index['data_size'].append(_datasize)
294                            self._index['messageNumber'].append(self.messages)
295                            self._index['isSubmessage'].append(_issubmessage)
296                            if _issubmessage:
297                                self._index['submessageOffset'].append(_submsgoffset)
298                                self._index['submessageBeginSection'].append(_submsgbegin)
299                            else:
300                                self._index['submessageOffset'].append(0)
301                                self._index['submessageBeginSection'].append(_submsgbegin)
302
303                            # Create Grib2Message with data.
304                            msg = Grib2Message(section0,section1,section2,section3,section4,section5,_bmapflag)
305                            msg._msgnum = self.messages-1
306                            msg._deflist = _deflist
307                            msg._coordlist = _coordlist
308                            if not no_data:
309                                shape = (msg.ny,msg.nx)
310                                ndim = 2
311                                if msg.typeOfValues == 0:
312                                    dtype = 'float32'
313                                elif msg.typeOfValues == 1:
314                                    dtype = 'int32'
315                                msg._data = Grib2MessageOnDiskArray(shape, ndim, dtype, self._filehandle,
316                                                                    msg, pos, _bmappos, _datapos)
317                            self._index['msg'].append(msg)
318
319                            break
320                        else:
321                            self._filehandle.seek(self._filehandle.tell()-4)
322                            self.messages += 1
323                            self._index['offset'].append(pos)
324                            self._index['bitmap_offset'].append(_bmappos)
325                            self._index['data_offset'].append(_datapos)
326                            self._index['size'].append(section0[-1])
327                            self._index['data_size'].append(_datasize)
328                            self._index['messageNumber'].append(self.messages)
329                            self._index['isSubmessage'].append(_issubmessage)
330                            self._index['submessageOffset'].append(_submsgoffset)
331                            self._index['submessageBeginSection'].append(_submsgbegin)
332
333                            # Create Grib2Message with data.
334                            msg = Grib2Message(section0,section1,section2,section3,section4,section5,_bmapflag)
335                            msg._msgnum = self.messages-1
336                            msg._deflist = _deflist
337                            msg._coordlist = _coordlist
338                            if not no_data:
339                                shape = (msg.ny,msg.nx)
340                                ndim = 2
341                                if msg.typeOfValues == 0:
342                                    dtype = 'float32'
343                                elif msg.typeOfValues == 1:
344                                    dtype = 'int32'
345                                msg._data = Grib2MessageOnDiskArray(shape, ndim, dtype, self._filehandle,
346                                                                    msg, pos, _bmappos, _datapos)
347                            self._index['msg'].append(msg)
348
349                            continue
350
351            except(struct.error):
352                if 'r' in self.mode:
353                    self._filehandle.seek(0)
354                break
355
356        # Index at end of _build_index()
357        if self._hasindex and not no_data:
358             self.variables = tuple(sorted(set([msg.shortName for msg in self._index['msg']])))
359             self.levels = tuple(sorted(set([msg.level for msg in self._index['msg']])))
360
361
362    def close(self):
363        """
364        Close the file handle
365        """
366        if not self._filehandle.closed:
367            self.messages = 0
368            self.current_message = 0
369            self._filehandle.close()
370            self.closed = self._filehandle.closed
371
372
373    def read(self, size=None):
374        """
375        Read size amount of GRIB2 messages from the current position. If no argument is
376        given, then size is None and all messages are returned from the current position
377        in the file. This read method follows the behavior of Python's builtin open()
378        function, but whereas that operates on units of bytes, we operate on units of
379        GRIB2 messages.
380
381        Parameters
382        ----------
383
384        **`size : int, optional`**
385
386        The number of GRIB2 messages to read from the current position. If no argument is
387        give, the default value is `None` and remainder of the file is read.
388
389        Returns
390        -------
391
392        `Grib2Message` object when size = 1 or a `list` of Grib2Messages when
393        size > 1.
394        """
395        if size is not None and size < 0:
396            size = None
397        if size is None or size > 1:
398            start = self.tell()
399            stop = self.messages if size is None else start+size
400            if size is None:
401                self.current_message = self.messages-1
402            else:
403                self.current_message += size
404            return self._index['msg'][slice(start,stop,1)]
405        elif size == 1:
406            self.current_message += 1
407            return self._index['msg'][self.current_message]
408        else:
409            None
410
411
412    def seek(self, pos):
413        """
414        Set the position within the file in units of GRIB2 messages.
415
416        Parameters
417        ----------
418
419        **`pos : int`**
420
421        The GRIB2 Message number to set the file pointer to.
422        """
423        if self._hasindex:
424            self._filehandle.seek(self._index['offset'][pos])
425            self.current_message = pos
426
427
428    def tell(self):
429        """
430        Returns the position of the file in units of GRIB2 Messages.
431        """
432        return self.current_message
433
434
435    def select(self,**kwargs):
436        """
437        Select GRIB2 messages by `Grib2Message` attributes.
438        """
439        # TODO: Added ability to process multiple values for each keyword (attribute)
440        idxs = []
441        nkeys = len(kwargs.keys())
442        for k,v in kwargs.items():
443            for m in self._index['msg']:
444                if hasattr(m,k) and getattr(m,k) == v: idxs.append(m._msgnum)
445        idxs = np.array(idxs,dtype=np.int32)
446        return [self._index['msg'][i] for i in [ii[0] for ii in collections.Counter(idxs).most_common() if ii[1] == nkeys]]
447
448
449    def write(self, msg):
450        """
451        Writes GRIB2 message object to file.
452
453        Parameters
454        ----------
455
456        **`msg : Grib2Message or sequence of Grib2Messages`**
457
458        GRIB2 message objects to write to file.
459        """
460        if isinstance(msg,list):
461            for m in msg:
462                self.write(m)
463            return
464
465        if issubclass(msg.__class__,_Grib2Message):
466            if hasattr(msg,'_msg'):
467                self._filehandle.write(msg._msg)
468            else:
469                if msg._signature != msg._generate_signature():
470                    msg.pack()
471                    self._filehandle.write(msg._msg)
472                else:
473                    if hasattr(msg._data,'filehandle'):
474                        msg._data.filehandle.seek(msg._data.offset)
475                        self._filehandle.write(msg._data.filehandle.read(msg.section0[-1]))
476                    else:
477                        msg.pack()
478                        self._filehandle.write(msg._msg)
479            self.flush()
480            self.size = os.path.getsize(self.name)
481            self._filehandle.seek(self.size-msg.section0[-1])
482            self._build_index()
483        else:
484            raise TypeError("msg must be a Grib2Message object.")
485        return
486
487
488    def flush(self):
489        """
490        Flush the file object buffer.
491        """
492        self._filehandle.flush()
493
494
495    def levels_by_var(self,name):
496        """
497        Return a list of level strings given a variable shortName.
498
499        Parameters
500        ----------
501
502        **`name : str`**
503
504        Grib2Message variable shortName
505
506        Returns
507        -------
508
509        A list of strings of unique level strings.
510        """
511        return list(sorted(set([msg.level for msg in self.select(shortName=name)])))
512
513
514    def vars_by_level(self,level):
515        """
516        Return a list of variable shortName strings given a level.
517
518        Parameters
519        ----------
520
521        **`level : str`**
522
523        Grib2Message variable level
524
525        Returns
526        -------
527
528        A list of strings of variable shortName strings.
529        """
530        return list(sorted(set([msg.shortName for msg in self.select(level=level)])))

533class Grib2Message:
534    """
535    """
536    def __new__(self, section0: np.array = np.array([struct.unpack('>I',b'GRIB')[0],0,0,2,0]),
537                      section1: np.array = np.zeros((13),dtype=np.int64),
538                      section2: bytes = None,
539                      section3: np.array = None,
540                      section4: np.array = None,
541                      section5: np.array = None, *args, **kwargs):
542
543        bases = list()
544        if section3 is None:
545            if 'gdtn' in kwargs.keys():
546                gdtn = kwargs['gdtn']
547                Gdt = templates.gdt_class_by_gdtn(gdtn)
548                bases.append(Gdt)
549                section3 = np.zeros((Gdt._len+5),dtype=np.int64)
550            else:
551                raise ValueError("Must provide GRIB2 Grid Definition Template Number or section 3 array")
552        else:
553            gdtn = section3[4]
554            Gdt = templates.gdt_class_by_gdtn(gdtn)
555            bases.append(Gdt)
556
557        if section4 is None:
558            if 'pdtn' in kwargs.keys():
559                pdtn = kwargs['pdtn']
560                Pdt = templates.pdt_class_by_pdtn(pdtn)
561                bases.append(Pdt)
562                section4 = np.zeros((Pdt._len+2),dtype=np.int64)
563            else:
564                raise ValueError("Must provide GRIB2 Production Definition Template Number or section 4 array")
565        else:
566            pdtn = section4[1]
567            Pdt = templates.pdt_class_by_pdtn(pdtn)
568            bases.append(Pdt)
569
570        if section5 is None:
571            if 'drtn' in kwargs.keys():
572                drtn = kwargs['drtn']
573                Drt = templates.drt_class_by_drtn(drtn)
574                bases.append(Drt)
575                section5 = np.zeros((Drt._len+2),dtype=np.int64)
576            else:
577                raise ValueError("Must provide GRIB2 Data Representation Template Number or section 5 array")
578        else:
579            drtn = section5[1]
580            Drt = templates.drt_class_by_drtn(drtn)
581            bases.append(Drt)
582
583        # attempt to use existing Msg class if it has already been made with gdtn,pdtn,drtn combo
584        try:
585            Msg = _msg_class_store[f"{gdtn}:{pdtn}:{drtn}"]
586        except KeyError:
587            @dataclass(init=False, repr=False)
588            class Msg(_Grib2Message, *bases):
589                pass
590            _msg_class_store[f"{gdtn}:{pdtn}:{drtn}"] = Msg
591
592
593
594        return Msg(section0, section1, section2, section3, section4, section5, *args)

 597@dataclass
 598class _Grib2Message:
 599    # GRIB2 Sections
 600    section0: np.array = field(init=True,repr=False)
 601    section1: np.array = field(init=True,repr=False)
 602    section2: bytes = field(init=True,repr=False)
 603    section3: np.array = field(init=True,repr=False)
 604    section4: np.array = field(init=True,repr=False)
 605    section5: np.array = field(init=True,repr=False)
 606    bitMapFlag: Grib2Metadata = field(init=True,repr=False,default=255)
 607
 608    # Section 0 looked up attributes
 609    indicatorSection: np.array = field(init=False,repr=False,default=templates.IndicatorSection())
 610    discipline: Grib2Metadata = field(init=False,repr=False,default=templates.Discipline())
 611
 612    # Section 1 looked up attributes
 613    identificationSection: np.array = field(init=False,repr=False,default=templates.IdentificationSection())
 614    originatingCenter: Grib2Metadata = field(init=False,repr=False,default=templates.OriginatingCenter())
 615    originatingSubCenter: Grib2Metadata = field(init=False,repr=False,default=templates.OriginatingSubCenter())
 616    masterTableInfo: Grib2Metadata = field(init=False,repr=False,default=templates.MasterTableInfo())
 617    localTableInfo: Grib2Metadata = field(init=False,repr=False,default=templates.LocalTableInfo())
 618    significanceOfReferenceTime: Grib2Metadata = field(init=False,repr=False,default=templates.SignificanceOfReferenceTime())
 619    year: int = field(init=False,repr=False,default=templates.Year())
 620    month: int = field(init=False,repr=False,default=templates.Month())
 621    day: int = field(init=False,repr=False,default=templates.Day())
 622    hour: int = field(init=False,repr=False,default=templates.Hour())
 623    minute: int = field(init=False,repr=False,default=templates.Minute())
 624    second: int = field(init=False,repr=False,default=templates.Second())
 625    refDate: datetime.datetime = field(init=False,repr=False,default=templates.RefDate())
 626    productionStatus: Grib2Metadata = field(init=False,repr=False,default=templates.ProductionStatus())
 627    typeOfData: Grib2Metadata = field(init=False,repr=False,default=templates.TypeOfData())
 628
 629    @property
 630    def _isNDFD(self):
 631        return np.all(self.section1[0:2]==[8,65535])
 632
 633    # Section 3 looked up common attributes.  Other looked up attributes are available according
 634    # to the Grid Definition Template.
 635    gridDefinitionSection: np.array = field(init=False,repr=False,default=templates.GridDefinitionSection())
 636    sourceOfGridDefinition: int = field(init=False,repr=False,default=templates.SourceOfGridDefinition())
 637    numberOfDataPoints: int = field(init=False,repr=False,default=templates.NumberOfDataPoints())
 638    interpretationOfListOfNumbers: Grib2Metadata = field(init=False,repr=False,default=templates.InterpretationOfListOfNumbers())
 639    gridDefinitionTemplateNumber: Grib2Metadata = field(init=False,repr=False,default=templates.GridDefinitionTemplateNumber())
 640    gridDefinitionTemplate: list = field(init=False,repr=False,default=templates.GridDefinitionTemplate())
 641    _earthparams: dict = field(init=False,repr=False,default=templates.EarthParams())
 642    _dxsign: float = field(init=False,repr=False,default=templates.DxSign())
 643    _dysign: float = field(init=False,repr=False,default=templates.DySign())
 644    _llscalefactor: float = field(init=False,repr=False,default=templates.LLScaleFactor())
 645    _lldivisor: float = field(init=False,repr=False,default=templates.LLDivisor())
 646    _xydivisor: float = field(init=False,repr=False,default=templates.XYDivisor())
 647    shapeOfEarth: Grib2Metadata = field(init=False,repr=False,default=templates.ShapeOfEarth())
 648    earthRadius: float = field(init=False,repr=False,default=templates.EarthRadius())
 649    earthMajorAxis: float = field(init=False,repr=False,default=templates.EarthMajorAxis())
 650    earthMinorAxis: float = field(init=False,repr=False,default=templates.EarthMinorAxis())
 651    resolutionAndComponentFlags: list = field(init=False,repr=False,default=templates.ResolutionAndComponentFlags())
 652    ny: int = field(init=False,repr=False,default=templates.Ny())
 653    nx: int = field(init=False,repr=False,default=templates.Nx())
 654    scanModeFlags: list = field(init=False,repr=False,default=templates.ScanModeFlags())
 655    projParameters: dict = field(init=False,repr=False,default=templates.ProjParameters())
 656
 657    # Section 4 attributes. Listed here are "extra" or "helper" attrs that use metadata from
 658    # the given PDT, but not a formal part of the PDT.
 659    productDefinitionTemplateNumber: Grib2Metadata = field(init=False,repr=False,default=templates.ProductDefinitionTemplateNumber())
 660    productDefinitionTemplate: np.array = field(init=False,repr=False,default=templates.ProductDefinitionTemplate())
 661    _varinfo: list = field(init=False, repr=False, default=templates.VarInfo())
 662    _fixedsfc1info: list = field(init=False, repr=False, default=templates.FixedSfc1Info())
 663    _fixedsfc2info: list = field(init=False, repr=False, default=templates.FixedSfc2Info())
 664    fullName: str = field(init=False, repr=False, default=templates.FullName())
 665    units: str = field(init=False, repr=False, default=templates.Units())
 666    shortName: str = field(init=False, repr=False, default=templates.ShortName())
 667    leadTime: datetime.timedelta = field(init=False,repr=False,default=templates.LeadTime())
 668    unitOfFirstFixedSurface: str = field(init=False,repr=False,default=templates.UnitOfFirstFixedSurface())
 669    valueOfFirstFixedSurface: int = field(init=False,repr=False,default=templates.ValueOfFirstFixedSurface())
 670    unitOfSecondFixedSurface: str = field(init=False,repr=False,default=templates.UnitOfSecondFixedSurface())
 671    valueOfSecondFixedSurface: int = field(init=False,repr=False,default=templates.ValueOfSecondFixedSurface())
 672    level: str = field(init=False, repr=False, default=templates.Level())
 673    duration: datetime.timedelta = field(init=False,repr=False,default=templates.Duration())
 674    validDate: datetime.datetime = field(init=False,repr=False,default=templates.ValidDate())
 675
 676    # Section 5 looked up common attributes.  Other looked up attributes are available according
 677    # to the Data Representation Template.
 678    numberOfPackedValues: int = field(init=False,repr=False,default=templates.NumberOfPackedValues())
 679    dataRepresentationTemplateNumber: Grib2Metadata = field(init=False,repr=False,default=templates.DataRepresentationTemplateNumber())
 680    dataRepresentationTemplate: list = field(init=False,repr=False,default=templates.DataRepresentationTemplate())
 681    typeOfValues: Grib2Metadata = field(init=False,repr=False,default=templates.TypeOfValues())
 682
 683
 684    def __post_init__(self):
 685        self._msgnum = -1
 686        self._deflist = None
 687        self._coordlist = None
 688        self._signature = self._generate_signature()
 689        try:
 690            self._sha1_section3 = hashlib.sha1(self.section3).hexdigest()
 691        except(TypeError):
 692            pass
 693        self.bitMapFlag = Grib2Metadata(self.bitMapFlag,table='6.0')
 694
 695
 696    @property
 697    def gdtn(self):
 698        return self.section3[4]
 699
 700
 701    @property
 702    def pdtn(self):
 703        return self.section4[1]
 704
 705
 706    @property
 707    def drtn(self):
 708        return self.section5[1]
 709
 710
 711    @property
 712    def pdy(self):
 713        return ''.join([str(i) for i in self.section1[5:8]])
 714
 715
 716    @property
 717    def griddef(self):
 718        return Grib2GridDef.from_section3(self.section3)
 719
 720
 721    def __repr__(self):
 722        info = ''
 723        for sect in [0,1,3,4,5,6]:
 724            for k,v in self.attrs_by_section(sect,values=True).items():
 725                info += f'Section {sect}: {k} = {v}\n'
 726        return info
 727
 728
 729    def __str__(self):
 730        return (f'{self._msgnum}:d={self.refDate}:{self.shortName}:'
 731                f'{self.fullName} ({self.units}):{self.level}:'
 732                f'{self.leadTime}')
 733
 734
 735    def _generate_signature(self):
 736        return hashlib.sha1(np.concatenate((self.section0,self.section1,
 737                                            self.section3,self.section4,
 738                                            self.section5))).hexdigest()
 739
 740
 741    def attrs_by_section(self, sect, values=False):
 742        """
 743        Provide a tuple of attribute names for the given GRIB2 section.
 744
 745        Parameters
 746        ----------
 747
 748        **`sect : int`**
 749
 750        The GRIB2 section number.
 751
 752        **`values : bool, optional`**
 753
 754        Optional (default is `False`) arugment to return attributes values.
 755
 756        Returns
 757        -------
 758
 759        A List attribute names or Dict if `values = True`.
 760        """
 761        if sect in {0,1,6}:
 762            attrs = templates._section_attrs[sect]
 763        elif sect in {3,4,5}:
 764            def _find_class_index(n):
 765                _key = {3:'Grid', 4:'Product', 5:'Data'}
 766                for i,c in enumerate(self.__class__.__mro__):
 767                    if _key[n] in c.__name__:
 768                        return i
 769                else:
 770                    return []
 771            if sys.version_info.minor <= 8:
 772                attrs = templates._section_attrs[sect]+\
 773                        [a for a in dir(self.__class__.__mro__[_find_class_index(sect)]) if not a.startswith('_')]
 774            else:
 775                attrs = templates._section_attrs[sect]+\
 776                        self.__class__.__mro__[_find_class_index(sect)]._attrs
 777        else:
 778            attrs = []
 779        if values:
 780            return {k:getattr(self,k) for k in attrs}
 781        else:
 782            return attrs
 783
 784
 785    def pack(self):
 786        """
 787        Packs GRIB2 section data into a binary message.  It is the user's responsibility
 788        to populate the GRIB2 section information with appropriate metadata.
 789        """
 790        # Create beginning of packed binary message with section 0 and 1 data.
 791        self._sections = []
 792        self._msg,self._pos = g2clib.grib2_create(self.indicatorSection[2:4],self.identificationSection)
 793        self._sections += [0,1]
 794
 795        # Add section 2 if present.
 796        if isinstance(self.section2,bytes) and len(self.section2) > 0:
 797            self._msg,self._pos = g2clib.grib2_addlocal(self._msg,self.section2)
 798            self._sections.append(2)
 799
 800        # Add section 3.
 801        self._msg,self._pos = g2clib.grib2_addgrid(self._msg,self.gridDefinitionSection,
 802                                                   self.gridDefinitionTemplate,
 803                                                   self._deflist)
 804        self._sections.append(3)
 805
 806        # Prepare data.
 807        field = np.copy(self.data)
 808        if self.scanModeFlags is not None:
 809            if self.scanModeFlags[3]:
 810                fieldsave = field.astype('f') # Casting makes a copy
 811                field[1::2,:] = fieldsave[1::2,::-1]
 812        fld = field.astype('f')
 813
 814        # Prepare bitmap, if necessary
 815        bitmapflag = self.bitMapFlag.value
 816        if bitmapflag == 0:
 817            bmap = np.ravel(np.where(np.isnan(fld),0,1)).astype(DEFAULT_NUMPY_INT)
 818        else:
 819            bmap = None
 820
 821        # Prepare optional coordinate list
 822        if self._coordlist is not None:
 823            crdlist = np.array(self._coordlist,'f')
 824        else:
 825            crdlist = None
 826
 827        # Prepare data for packing if nans are present
 828        fld = np.ravel(fld)
 829        if np.isnan(fld).any() and hasattr(self,'_missvalmap'):
 830            fld = np.where(self._missvalmap==1,self.priMissingValue,fld)
 831            fld = np.where(self._missvalmap==2,self.secMissingValue,fld)
 832
 833        # Add sections 4, 5, 6 (if present), and 7.
 834        self._msg,self._pos = g2clib.grib2_addfield(self._msg,self.pdtn,
 835                                                    self.productDefinitionTemplate,
 836                                                    crdlist,
 837                                                    self.drtn,
 838                                                    self.dataRepresentationTemplate,
 839                                                    fld,
 840                                                    bitmapflag,
 841                                                    bmap)
 842        self._sections.append(4)
 843        self._sections.append(5)
 844        if bmap is not None: self._sections.append(6)
 845        self._sections.append(7)
 846
 847        # Finalize GRIB2 message with section 8.
 848        self._msg, self._pos = g2clib.grib2_end(self._msg)
 849        self._sections.append(8)
 850        self.section0[-1] = len(self._msg)
 851
 852
 853    @property
 854    def data(self) -> np.array:
 855        """
 856        Accessing the data attribute loads data into memmory
 857        """
 858        if not hasattr(self,'_auto_nans'): self._auto_nans = _AUTO_NANS
 859        if hasattr(self,'_data'):
 860            if self._auto_nans != _AUTO_NANS:
 861                self._data = self._ondiskarray
 862            if isinstance(self._data, Grib2MessageOnDiskArray):
 863                self._ondiskarray = self._data
 864                self._data = np.asarray(self._data)
 865            return self._data
 866        raise ValueError
 867
 868    @data.setter
 869    def data(self, data):
 870        if not isinstance(data, np.ndarray):
 871            raise ValueError('Grib2Message data only supports numpy arrays')
 872        self._data = data
 873
 874
 875    def __getitem__(self, item):
 876        return self.data[item]
 877
 878
 879    def __setitem__(self, item):
 880        raise NotImplementedError('assignment of data not supported via setitem')
 881
 882
 883    def latlons(self, *args, **kwrgs):
 884        """Alias for `grib2io.Grib2Message.grid` method"""
 885        return self.grid(*args, **kwrgs)
 886
 887
 888    def grid(self, unrotate=True):
 889        """
 890        Return lats,lons (in degrees) of grid. Currently can handle reg. lat/lon,
 891        global Gaussian, mercator, stereographic, lambert conformal, albers equal-area,
 892        space-view and azimuthal equidistant grids.
 893
 894        Parameters
 895        ----------
 896
 897        **`unrotate : bool`**
 898
 899        If `True` [DEFAULT], and grid is rotated lat/lon, then unrotate the grid,
 900        otherwise `False`, do not.
 901
 902        Returns
 903        -------
 904
 905        **`lats, lons : numpy.ndarray`**
 906
 907        Returns two numpy.ndarrays with dtype=numpy.float32 of grid latitudes and
 908        longitudes in units of degrees.
 909        """
 910        if self._sha1_section3 in _latlon_datastore.keys():
 911            return _latlon_datastore[self._sha1_section3]
 912        gdtn = self.gridDefinitionTemplateNumber.value
 913        gdtmpl = self.gridDefinitionTemplate
 914        reggrid = self.gridDefinitionSection[2] == 0 # This means regular 2-d grid
 915        if gdtn == 0:
 916            # Regular lat/lon grid
 917            lon1, lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 918            lon2, lat2 = self.longitudeLastGridpoint, self.latitudeLastGridpoint
 919            dlon = self.gridlengthXDirection
 920            dlat = self.gridlengthYDirection
 921            if reggrid:
 922                lats = np.arange(lat1,lat2+dlat,dlat)
 923                lons = np.arange(lon1,lon2+dlon,dlon)
 924            else:
 925                lats = np.linspace(lat1,lat2,self.ny)
 926                lons = np.linspace(lon1,lon2,self.ny*2)
 927            lons,lats = np.meshgrid(lons,lats) # Make 2-d arrays.
 928        elif gdtn == 1: # Rotated Lat/Lon grid
 929            pj = pyproj.Proj(self.projParameters)
 930            lat1,lon1 = self.latitudeFirstGridpoint,self.longitudeFirstGridpoint
 931            lat2,lon2 = self.latitudeLastGridpoint,self.longitudeLastGridpoint
 932            if lon1 > 180.0: lon1 -= 360.0
 933            if lon2 > 180.0: lon2 -= 360.0
 934            lats = np.linspace(lat1,lat2,self.ny)
 935            lons = np.linspace(lon1,lon2,self.nx)
 936            lons,lats = np.meshgrid(lons,lats) # Make 2-d arrays.
 937            if unrotate:
 938                from grib2io.utils import rotated_grid
 939                lats,lons = rotated_grid.unrotate(lats,lons,self.anglePoleRotation,
 940                                                  self.latitudeSouthernPole,
 941                                                  self.longitudeSouthernPole)
 942        elif gdtn == 40: # Gaussian grid (only works for global!)
 943            from utils.gauss_grids import gaussian_latitudes
 944            lon1, lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 945            lon2, lat2 = self.longitudeLastGridpoint, self.latitudeLastGridpoint
 946            nlats = self.ny
 947            if not reggrid: # Reduced Gaussian grid.
 948                nlons = 2*nlats
 949                dlon = 360./nlons
 950            else:
 951                nlons = self.nx
 952                dlon = self.gridlengthXDirection
 953            lons = np.arange(lon1,lon2+dlon,dlon)
 954            # Compute Gaussian lats (north to south)
 955            lats = gaussian_latitudes(nlats)
 956            if lat1 < lat2:  # reverse them if necessary
 957                lats = lats[::-1]
 958            lons,lats = np.meshgrid(lons,lats)
 959        elif gdtn in {10,20,30,31,110}:
 960            # Mercator, Lambert Conformal, Stereographic, Albers Equal Area, Azimuthal Equidistant
 961            dx,dy = self.gridlengthXDirection, self.gridlengthYDirection
 962            lon1,lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 963            pj = pyproj.Proj(self.projParameters)
 964            llcrnrx, llcrnry = pj(lon1,lat1)
 965            x = llcrnrx+dx*np.arange(self.nx)
 966            y = llcrnry+dy*np.arange(self.ny)
 967            x,y = np.meshgrid(x, y)
 968            lons,lats = pj(x, y, inverse=True)
 969        elif gdtn == 90:
 970            # Satellite Projection
 971            dx = self.gridlengthXDirection
 972            dy = self.gridlengthYDirection
 973            pj = pyproj.Proj(self.projParameters)
 974            x = dx*np.indices((self.ny,self.nx),'f')[1,:,:]
 975            x -= 0.5*x.max()
 976            y = dy*np.indices((self.ny,self.nx),'f')[0,:,:]
 977            y -= 0.5*y.max()
 978            lons,lats = pj(x,y,inverse=True)
 979            # Set lons,lats to 1.e30 where undefined
 980            abslons = np.fabs(lons)
 981            abslats = np.fabs(lats)
 982            lons = np.where(abslons < 1.e20, lons, 1.e30)
 983            lats = np.where(abslats < 1.e20, lats, 1.e30)
 984        else:
 985            raise ValueError('Unsupported grid')
 986
 987        _latlon_datastore[self._sha1_section3] = (lats,lons)
 988
 989        return lats, lons
 990
 991
 992    def map_keys(self):
 993        """
 994        Returns an unpacked data grid where integer grid values are replaced with
 995        a string in which the numeric value is a representation of. These types
 996        of fields are cateogrical or classifications where data values do not
 997        represent an observable or predictable physical quantity.
 998
 999        An example of such a field field would be [Dominant Precipitation Type -
1000        DPTYPE](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_table4-201.shtml)
1001
1002        Returns
1003        -------
1004
1005        **`numpy.ndarray`** of string values per element.
1006        """
1007        hold_auto_nans = _AUTO_NANS
1008        set_auto_nans(False)
1009        if (np.all(self.section1[0:2]==[7,14]) and self.shortName == 'PWTHER') or \
1010        (np.all(self.section1[0:2]==[8,65535]) and self.shortName == 'WX'):
1011            keys = utils.decode_wx_strings(self.section2)
1012            if hasattr(self,'priMissingValue') and self.priMissingValue not in [None,0]:
1013                keys[int(self.priMissingValue)] = 'Missing'
1014            if hasattr(self,'secMissingValue') and self.secMissingValue not in [None,0]:
1015                keys[int(self.secMissingValue)] = 'Missing'
1016            u,inv = np.unique(self.data,return_inverse=True)
1017            fld = np.array([keys[x] for x in u])[inv].reshape(self.data.shape)
1018        else:
1019            # For data whose units are defined in a code table
1020            tbl = re.findall(r'\d\.\d+',self.units,re.IGNORECASE)[0]
1021            for k,v in tables.get_table(tbl).items():
1022                fld = np.where(fld==k,v,fld)
1023        set_auto_nans(hold_auto_nans)
1024        return fld
1025
1026
1027    def to_bytes(self, validate=True):
1028        """
1029        Return packed GRIB2 message in bytes format. This will be Useful for
1030        exporting data in non-file formats.  For example, can be used to
1031        output grib data directly to S3 using the boto3 client without the
1032        need to write a temporary file to upload first.
1033
1034        Parameters
1035        ----------
1036
1037        **`validate : bool, optional`**
1038
1039        If `True` (DEFAULT), validates first/last four bytes for proper
1040        formatting, else returns None. If `False`, message is output as is.
1041
1042        Returns
1043        -------
1044
1045        Returns GRIB2 formatted message as bytes.
1046        """
1047        if validate:
1048            if self._msg[0:4]+self._msg[-4:] == b'GRIB7777':
1049                return self._msg
1050            else:
1051                return None
1052        else:
1053            return self._msg
1054
1055
1056    def interpolate(self, method, grid_def_out, method_options=None):
1057        """
1058        Perform grid spatial interpolation via the [NCEPLIBS-ip library](https://github.com/NOAA-EMC/NCEPLIBS-ip).
1059
1060        Parameters
1061        ----------
1062
1063        **`method : int or str`**
1064
1065        Interpolate method to use. This can either be an integer or string using
1066        the following mapping:
1067
1068        | Interpolate Scheme | Integer Value |
1069        | :---:              | :---:         |
1070        | 'bilinear'         | 0             |
1071        | 'bicubic'          | 1             |
1072        | 'neighbor'         | 2             |
1073        | 'budget'           | 3             |
1074        | 'spectral'         | 4             |
1075        | 'neighbor-budget'  | 6             |
1076
1077        **`grid_def_out : grib2io.Grib2GridDef`**
1078
1079        Grib2GridDef object of the output grid.
1080
1081        **`method_options : list of ints, optional`**
1082
1083        Interpolation options. See the NCEPLIBS-ip doucmentation for
1084        more information on how these are used.
1085        """
1086        section0 = self.section0
1087        section0[-1] = 0
1088        gds = [0, grid_def_out.nx*grid_def_out.ny, 0, 255, grid_def_out.gdtn]
1089        section3 = np.concatenate((gds,grid_def_out.gdt))
1090
1091        msg = Grib2Message(section0,
1092                           self.section1,
1093                           self.section2,
1094                           section3,
1095                           self.section4,
1096                           self.section5,
1097                           self.bitMapFlag.value)
1098        msg._msgnum = -1
1099        msg._deflist = self._deflist
1100        msg._coordlist = self._coordlist
1101        shape = (msg.ny,msg.nx)
1102        ndim = 2
1103        if msg.typeOfValues == 0:
1104            dtype = 'float32'
1105        elif msg.typeOfValues == 1:
1106            dtype = 'int32'
1107        msg._data = interpolate(self.data,method,Grib2GridDef.from_section3(self.section3),grid_def_out,
1108                                method_options=method_options).reshape(msg.ny,msg.nx)
1109        return msg