52class open():
 53    """
 54    GRIB2 File Object.  A physical file can contain one or more GRIB2 messages.  When instantiated,
 55    class `grib2io.open`, the file named `filename` is opened for reading (`mode = 'r'`) and is
 56    automatically indexed.  The indexing procedure reads some of the GRIB2 metadata for all GRIB2 Messages.
 57
 58    A GRIB2 Message may contain submessages whereby Section 2-7 can be repeated.  grib2io accommodates
 59    for this by flattening any GRIB2 submessages into multiple individual messages.
 60
 61    Attributes
 62    ----------
 63
 64    **`mode`** File IO mode of opening the file.
 65
 66    **`name`** Full path name of the GRIB2 file.
 67
 68    **`messages`** Count of GRIB2 Messages contained in the file.
 69
 70    **`current_message`** Current position of the file in units of GRIB2 Messages.
 71
 72    **`size`** Size of the file in units of bytes.
 73
 74    **`closed`** `True` is file handle is close; `False` otherwise.
 75
 76    **`variables`** Tuple containing a unique list of variable short names (i.e. GRIB2 abbreviation names).
 77
 78    **`levels`** Tuple containing a unique list of wgrib2-formatted level/layer strings.
 79    """
 80    __slots__ = ('_filehandle','_hasindex','_index','mode','name','messages',
 81                 'current_message','size','closed','variables','levels','_pos')
 82    def __init__(self, filename, mode='r', **kwargs):
 83        """
 84        `open` Constructor
 85
 86        Parameters
 87        ----------
 88
 89        **`filename : str`**
 90
 91        File name containing GRIB2 messages.
 92
 93        **`mode : str, optional`**
 94
 95        File access mode where `r` opens the files for reading only; `w` opens the file for writing.
 96        """
 97        if mode in {'a','r','w'}:
 98            mode = mode+'b'
 99            if 'w' in mode: mode += '+'
100        self._filehandle = builtins.open(filename,mode=mode,buffering=ONE_MB)
101        self._hasindex = False
102        self._index = {}
103        self.mode = mode
104        self.name = os.path.abspath(filename)
105        self.messages = 0
106        self.current_message = 0
107        self.size = os.path.getsize(self.name)
108        self.closed = self._filehandle.closed
109        self.levels = None
110        self.variables = None
111        if 'r' in self.mode:
112            try:
113                self._build_index(no_data=kwargs['_xarray_backend'])
114            except(KeyError):
115                self._build_index()
116        # FIX: Cannot perform reads on mode='a'
117        #if 'a' in self.mode and self.size > 0: self._build_index()
118
119
120    def __delete__(self, instance):
121        """
122        """
123        self.close()
124        del self._index
125
126
127    def __enter__(self):
128        """
129        """
130        return self
131
132
133    def __exit__(self, atype, value, traceback):
134        """
135        """
136        self.close()
137
138
139    def __iter__(self):
140        """
141        """
142        yield from self._index['msg']
143
144
145    def __len__(self):
146        """
147        """
148        return self.messages
149
150
151    def __repr__(self):
152        """
153        """
154        strings = []
155        for k in self.__slots__:
156            if k.startswith('_'): continue
157            strings.append('%s = %s\n'%(k,eval('self.'+k)))
158        return ''.join(strings)
159
160
161    def __getitem__(self, key):
162        """
163        """
164        if isinstance(key,int):
165            if abs(key) >= len(self._index['msg']):
166                raise IndexError("index out of range")
167            else:
168                return self._index['msg'][key]
169        elif isinstance(key,str):
170            return self.select(shortName=key)
171        elif isinstance(key,slice):
172            return self._index['msg'][key]
173        else:
174            raise KeyError('Key must be an integer, slice, or GRIB2 variable shortName.')
175
176
177    def _build_index(self, no_data=False):
178        """
179        Perform indexing of GRIB2 Messages.
180        """
181        # Initialize index dictionary
182        if not self._hasindex:
183            self._index['offset'] = []
184            self._index['bitmap_offset'] = []
185            self._index['data_offset'] = []
186            self._index['size'] = []
187            self._index['data_size'] = []
188            self._index['submessageOffset'] = []
189            self._index['submessageBeginSection'] = []
190            self._index['isSubmessage'] = []
191            self._index['messageNumber'] = []
192            self._index['msg'] = []
193            self._hasindex = True
194
195        # Iterate
196        while True:
197            try:
198                # Read first 4 bytes and decode...looking for "GRIB"
199                pos = self._filehandle.tell()
200                header = struct.unpack('>i',self._filehandle.read(4))[0]
201
202                # Test header. Then get information from GRIB2 Section 0: the discipline
203                # number, edition number (should always be 2), and GRIB2 message size.
204                # Then iterate to check for submessages.
205                if header.to_bytes(4,'big') == b'GRIB':
206
207                    _issubmessage = False
208                    _submsgoffset = 0
209                    _submsgbegin = 0
210                    _bmapflag = None
211
212                    # Read the rest of Section 0 using struct.
213                    section0 = np.concatenate(([header],list(struct.unpack('>HBBQ',self._filehandle.read(12)))),dtype=np.int64)
214                    assert section0[3] == 2
215
216                    # Read and unpack Section 1
217                    secsize = struct.unpack('>i',self._filehandle.read(4))[0]
218                    secnum = struct.unpack('>B',self._filehandle.read(1))[0]
219                    assert secnum == 1
220                    self._filehandle.seek(self._filehandle.tell()-5)
221                    _grbmsg = self._filehandle.read(secsize)
222                    _grbpos = 0
223                    section1,_grbpos = g2clib.unpack1(_grbmsg,_grbpos,np.empty)
224                    secrange = range(2,8)
225                    while 1:
226                        section2 = b''
227                        for num in secrange:
228                            secsize = struct.unpack('>i',self._filehandle.read(4))[0]
229                            secnum = struct.unpack('>B',self._filehandle.read(1))[0]
230                            if secnum == num:
231                                if secnum == 2:
232                                    if secsize > 0:
233                                        section2 = self._filehandle.read(secsize-5)
234                                elif secnum == 3:
235                                    self._filehandle.seek(self._filehandle.tell()-5)
236                                    _grbmsg = self._filehandle.read(secsize)
237                                    _grbpos = 0
238                                    # Unpack Section 3
239                                    _gds,_gdt,_deflist,_grbpos = g2clib.unpack3(_grbmsg,_grbpos,np.empty)
240                                    _gds = _gds.tolist()
241                                    _gdt = _gdt.tolist()
242                                    section3 = np.concatenate((_gds,_gdt))
243                                    section3 = np.where(section3==4294967295,-1,section3)
244                                elif secnum == 4:
245                                    self._filehandle.seek(self._filehandle.tell()-5)
246                                    _grbmsg = self._filehandle.read(secsize)
247                                    _grbpos = 0
248                                    # Unpack Section 4
249                                    _numcoord,_pdt,_pdtnum,_coordlist,_grbpos = g2clib.unpack4(_grbmsg,_grbpos,np.empty)
250                                    _pdt = _pdt.tolist()
251                                    section4 = np.concatenate((np.array((_numcoord,_pdtnum)),_pdt))
252                                elif secnum == 5:
253                                    self._filehandle.seek(self._filehandle.tell()-5)
254                                    _grbmsg = self._filehandle.read(secsize)
255                                    _grbpos = 0
256                                    # Unpack Section 5
257                                    _drt,_drtn,_npts,self._pos = g2clib.unpack5(_grbmsg,_grbpos,np.empty)
258                                    section5 = np.concatenate((np.array((_npts,_drtn)),_drt))
259                                elif secnum == 6:
260                                    # Unpack Section 6. Not really...just get the flag value.
261                                    _bmapflag = struct.unpack('>B',self._filehandle.read(1))[0]
262                                    if _bmapflag == 0:
263                                        _bmappos = self._filehandle.tell()-6
264                                    elif _bmapflag == 254:
265                                        pass # Do this to keep the previous position value
266                                    else:
267                                        _bmappos = None
268                                    self._filehandle.seek(self._filehandle.tell()+secsize-6)
269                                elif secnum == 7:
270                                    # Unpack Section 7. No need to read it, just index the position in file.
271                                    _datapos = self._filehandle.tell()-5
272                                    _datasize = secsize
273                                    self._filehandle.seek(self._filehandle.tell()+secsize-5)
274                                else:
275                                    self._filehandle.seek(self._filehandle.tell()+secsize-5)
276                            else:
277                                if num == 2 and secnum == 3:
278                                    pass # Allow this.  Just means no Local Use Section.
279                                else:
280                                    _issubmessage = True
281                                    _submsgoffset = (self._filehandle.tell()-5)-(self._index['offset'][-1])
282                                    _submsgbegin = secnum
283                                self._filehandle.seek(self._filehandle.tell()-5)
284                                continue
285                        trailer = struct.unpack('>4s',self._filehandle.read(4))[0]
286                        if trailer == b'7777':
287                            self.messages += 1
288                            self._index['offset'].append(pos)
289                            self._index['bitmap_offset'].append(_bmappos)
290                            self._index['data_offset'].append(_datapos)
291                            self._index['size'].append(section0[-1])
292                            self._index['data_size'].append(_datasize)
293                            self._index['messageNumber'].append(self.messages)
294                            self._index['isSubmessage'].append(_issubmessage)
295                            if _issubmessage:
296                                self._index['submessageOffset'].append(_submsgoffset)
297                                self._index['submessageBeginSection'].append(_submsgbegin)
298                            else:
299                                self._index['submessageOffset'].append(0)
300                                self._index['submessageBeginSection'].append(_submsgbegin)
301
302                            # Create Grib2Message with data.
303                            msg = Grib2Message(section0,section1,section2,section3,section4,section5,_bmapflag)
304                            msg._msgnum = self.messages-1
305                            msg._deflist = _deflist
306                            msg._coordlist = _coordlist
307                            if not no_data:
308                                shape = (msg.ny,msg.nx)
309                                ndim = 2
310                                if msg.typeOfValues == 0:
311                                    dtype = 'float32'
312                                elif msg.typeOfValues == 1:
313                                    dtype = 'int32'
314                                msg._data = Grib2MessageOnDiskArray(shape, ndim, dtype, self._filehandle,
315                                                                    msg, pos, _bmappos, _datapos)
316                            self._index['msg'].append(msg)
317
318                            break
319                        else:
320                            self._filehandle.seek(self._filehandle.tell()-4)
321                            self.messages += 1
322                            self._index['offset'].append(pos)
323                            self._index['bitmap_offset'].append(_bmappos)
324                            self._index['data_offset'].append(_datapos)
325                            self._index['size'].append(section0[-1])
326                            self._index['data_size'].append(_datasize)
327                            self._index['messageNumber'].append(self.messages)
328                            self._index['isSubmessage'].append(_issubmessage)
329                            self._index['submessageOffset'].append(_submsgoffset)
330                            self._index['submessageBeginSection'].append(_submsgbegin)
331
332                            # Create Grib2Message with data.
333                            msg = Grib2Message(section0,section1,section2,section3,section4,section5,_bmapflag)
334                            msg._msgnum = self.messages-1
335                            msg._deflist = _deflist
336                            msg._coordlist = _coordlist
337                            if not no_data:
338                                shape = (msg.ny,msg.nx)
339                                ndim = 2
340                                if msg.typeOfValues == 0:
341                                    dtype = 'float32'
342                                elif msg.typeOfValues == 1:
343                                    dtype = 'int32'
344                                msg._data = Grib2MessageOnDiskArray(shape, ndim, dtype, self._filehandle,
345                                                                    msg, pos, _bmappos, _datapos)
346                            self._index['msg'].append(msg)
347
348                            continue
349
350            except(struct.error):
351                if 'r' in self.mode:
352                    self._filehandle.seek(0)
353                break
354
355        # Index at end of _build_index()
356        if self._hasindex and not no_data:
357             self.variables = tuple(sorted(set([msg.shortName for msg in self._index['msg']])))
358             self.levels = tuple(sorted(set([msg.level for msg in self._index['msg']])))
359
360
361    def close(self):
362        """
363        Close the file handle
364        """
365        if not self._filehandle.closed:
366            self.messages = 0
367            self.current_message = 0
368            self._filehandle.close()
369            self.closed = self._filehandle.closed
370
371
372    def read(self, size=None):
373        """
374        Read size amount of GRIB2 messages from the current position. If no argument is
375        given, then size is None and all messages are returned from the current position
376        in the file. This read method follows the behavior of Python's builtin open()
377        function, but whereas that operates on units of bytes, we operate on units of
378        GRIB2 messages.
379
380        Parameters
381        ----------
382
383        **`size : int, optional`**
384
385        The number of GRIB2 messages to read from the current position. If no argument is
386        give, the default value is `None` and remainder of the file is read.
387
388        Returns
389        -------
390
391        `Grib2Message` object when size = 1 or a `list` of Grib2Messages when
392        size > 1.
393        """
394        if size is not None and size < 0:
395            size = None
396        if size is None or size > 1:
397            start = self.tell()
398            stop = self.messages if size is None else start+size
399            if size is None:
400                self.current_message = self.messages-1
401            else:
402                self.current_message += size
403            return self._index['msg'][slice(start,stop,1)]
404        elif size == 1:
405            self.current_message += 1
406            return self._index['msg'][self.current_message]
407        else:
408            None
409
410
411    def seek(self, pos):
412        """
413        Set the position within the file in units of GRIB2 messages.
414
415        Parameters
416        ----------
417
418        **`pos : int`**
419
420        The GRIB2 Message number to set the file pointer to.
421        """
422        if self._hasindex:
423            self._filehandle.seek(self._index['offset'][pos])
424            self.current_message = pos
425
426
427    def tell(self):
428        """
429        Returns the position of the file in units of GRIB2 Messages.
430        """
431        return self.current_message
432
433
434    def select(self,**kwargs):
435        """
436        Select GRIB2 messages by `Grib2Message` attributes.
437        """
438        # TODO: Added ability to process multiple values for each keyword (attribute)
439        idxs = []
440        nkeys = len(kwargs.keys())
441        for k,v in kwargs.items():
442            for m in self._index['msg']:
443                if hasattr(m,k) and getattr(m,k) == v: idxs.append(m._msgnum)
444        idxs = np.array(idxs,dtype=np.int32)
445        return [self._index['msg'][i] for i in [ii[0] for ii in collections.Counter(idxs).most_common() if ii[1] == nkeys]]
446
447
448    def write(self, msg):
449        """
450        Writes GRIB2 message object to file.
451
452        Parameters
453        ----------
454
455        **`msg : Grib2Message or sequence of Grib2Messages`**
456
457        GRIB2 message objects to write to file.
458        """
459        if isinstance(msg,list):
460            for m in msg:
461                self.write(m)
462            return
463
464        if issubclass(msg.__class__,_Grib2Message):
465            if hasattr(msg,'_msg'):
466                self._filehandle.write(msg._msg)
467            else:
468                if msg._signature != msg._generate_signature():
469                    msg.pack()
470                    self._filehandle.write(msg._msg)
471                else:
472                    if hasattr(msg._data,'filehandle'):
473                        msg._data.filehandle.seek(msg._data.offset)
474                        self._filehandle.write(msg._data.filehandle.read(msg.section0[-1]))
475                    else:
476                        msg.pack()
477                        self._filehandle.write(msg._msg)
478            self.flush()
479            self.size = os.path.getsize(self.name)
480            self._filehandle.seek(self.size-msg.section0[-1])
481            self._build_index()
482        else:
483            raise TypeError("msg must be a Grib2Message object.")
484        return
485
486
487    def flush(self):
488        """
489        Flush the file object buffer.
490        """
491        self._filehandle.flush()
492
493
494    def levels_by_var(self,name):
495        """
496        Return a list of level strings given a variable shortName.
497
498        Parameters
499        ----------
500
501        **`name : str`**
502
503        Grib2Message variable shortName
504
505        Returns
506        -------
507
508        A list of strings of unique level strings.
509        """
510        return list(sorted(set([msg.level for msg in self.select(shortName=name)])))
511
512
513    def vars_by_level(self,level):
514        """
515        Return a list of variable shortName strings given a level.
516
517        Parameters
518        ----------
519
520        **`level : str`**
521
522        Grib2Message variable level
523
524        Returns
525        -------
526
527        A list of strings of variable shortName strings.
528        """
529        return list(sorted(set([msg.shortName for msg in self.select(level=level)])))

532class Grib2Message:
533    """
534    """
535    def __new__(self, section0: np.array = np.array([struct.unpack('>I',b'GRIB')[0],0,0,2,0]),
536                      section1: np.array = np.zeros((13),dtype=np.int64),
537                      section2: bytes = None,
538                      section3: np.array = None,
539                      section4: np.array = None,
540                      section5: np.array = None, *args, **kwargs):
541
542        bases = list()
543        if section3 is None:
544            if 'gdtn' in kwargs.keys():
545                gdtn = kwargs['gdtn']
546                Gdt = templates.gdt_class_by_gdtn(gdtn)
547                bases.append(Gdt)
548                section3 = np.zeros((Gdt._len+5),dtype=np.int64)
549            else:
550                raise ValueError("Must provide GRIB2 Grid Definition Template Number or section 3 array")
551        else:
552            gdtn = section3[4]
553            Gdt = templates.gdt_class_by_gdtn(gdtn)
554            bases.append(Gdt)
555
556        if section4 is None:
557            if 'pdtn' in kwargs.keys():
558                pdtn = kwargs['pdtn']
559                Pdt = templates.pdt_class_by_pdtn(pdtn)
560                bases.append(Pdt)
561                section4 = np.zeros((Pdt._len+2),dtype=np.int64)
562            else:
563                raise ValueError("Must provide GRIB2 Production Definition Template Number or section 4 array")
564        else:
565            pdtn = section4[1]
566            Pdt = templates.pdt_class_by_pdtn(pdtn)
567            bases.append(Pdt)
568
569        if section5 is None:
570            if 'drtn' in kwargs.keys():
571                drtn = kwargs['drtn']
572                Drt = templates.drt_class_by_drtn(drtn)
573                bases.append(Drt)
574                section5 = np.zeros((Drt._len+2),dtype=np.int64)
575            else:
576                raise ValueError("Must provide GRIB2 Data Representation Template Number or section 5 array")
577        else:
578            drtn = section5[1]
579            Drt = templates.drt_class_by_drtn(drtn)
580            bases.append(Drt)
581
582        # attempt to use existing Msg class if it has already been made with gdtn,pdtn,drtn combo
583        try:
584            Msg = _msg_class_store[f"{gdtn}:{pdtn}:{drtn}"]
585        except KeyError:
586            @dataclass(init=False, repr=False)
587            class Msg(_Grib2Message, *bases):
588                pass
589            _msg_class_store[f"{gdtn}:{pdtn}:{drtn}"] = Msg
590
591
592
593        return Msg(section0, section1, section2, section3, section4, section5, *args)

 596@dataclass
 597class _Grib2Message:
 598    # GRIB2 Sections
 599    section0: np.array = field(init=True,repr=False)
 600    section1: np.array = field(init=True,repr=False)
 601    section2: bytes = field(init=True,repr=False)
 602    section3: np.array = field(init=True,repr=False)
 603    section4: np.array = field(init=True,repr=False)
 604    section5: np.array = field(init=True,repr=False)
 605    bitMapFlag: Grib2Metadata = field(init=True,repr=False,default=255)
 606
 607    # Section 0 looked up attributes
 608    indicatorSection: np.array = field(init=False,repr=False,default=templates.IndicatorSection())
 609    discipline: Grib2Metadata = field(init=False,repr=False,default=templates.Discipline())
 610
 611    # Section 1 looked up attributes
 612    identificationSection: np.array = field(init=False,repr=False,default=templates.IdentificationSection())
 613    originatingCenter: Grib2Metadata = field(init=False,repr=False,default=templates.OriginatingCenter())
 614    originatingSubCenter: Grib2Metadata = field(init=False,repr=False,default=templates.OriginatingSubCenter())
 615    masterTableInfo: Grib2Metadata = field(init=False,repr=False,default=templates.MasterTableInfo())
 616    localTableInfo: Grib2Metadata = field(init=False,repr=False,default=templates.LocalTableInfo())
 617    significanceOfReferenceTime: Grib2Metadata = field(init=False,repr=False,default=templates.SignificanceOfReferenceTime())
 618    year: int = field(init=False,repr=False,default=templates.Year())
 619    month: int = field(init=False,repr=False,default=templates.Month())
 620    day: int = field(init=False,repr=False,default=templates.Day())
 621    hour: int = field(init=False,repr=False,default=templates.Hour())
 622    minute: int = field(init=False,repr=False,default=templates.Minute())
 623    second: int = field(init=False,repr=False,default=templates.Second())
 624    refDate: datetime.datetime = field(init=False,repr=False,default=templates.RefDate())
 625    productionStatus: Grib2Metadata = field(init=False,repr=False,default=templates.ProductionStatus())
 626    typeOfData: Grib2Metadata = field(init=False,repr=False,default=templates.TypeOfData())
 627
 628    @property
 629    def _isNDFD(self):
 630        return np.all(self.section1[0:2]==[8,65535])
 631
 632    # Section 3 looked up common attributes.  Other looked up attributes are available according
 633    # to the Grid Definition Template.
 634    gridDefinitionSection: np.array = field(init=False,repr=False,default=templates.GridDefinitionSection())
 635    sourceOfGridDefinition: int = field(init=False,repr=False,default=templates.SourceOfGridDefinition())
 636    numberOfDataPoints: int = field(init=False,repr=False,default=templates.NumberOfDataPoints())
 637    interpretationOfListOfNumbers: Grib2Metadata = field(init=False,repr=False,default=templates.InterpretationOfListOfNumbers())
 638    gridDefinitionTemplateNumber: Grib2Metadata = field(init=False,repr=False,default=templates.GridDefinitionTemplateNumber())
 639    gridDefinitionTemplate: list = field(init=False,repr=False,default=templates.GridDefinitionTemplate())
 640    _earthparams: dict = field(init=False,repr=False,default=templates.EarthParams())
 641    _dxsign: float = field(init=False,repr=False,default=templates.DxSign())
 642    _dysign: float = field(init=False,repr=False,default=templates.DySign())
 643    _llscalefactor: float = field(init=False,repr=False,default=templates.LLScaleFactor())
 644    _lldivisor: float = field(init=False,repr=False,default=templates.LLDivisor())
 645    _xydivisor: float = field(init=False,repr=False,default=templates.XYDivisor())
 646    shapeOfEarth: Grib2Metadata = field(init=False,repr=False,default=templates.ShapeOfEarth())
 647    earthRadius: float = field(init=False,repr=False,default=templates.EarthRadius())
 648    earthMajorAxis: float = field(init=False,repr=False,default=templates.EarthMajorAxis())
 649    earthMinorAxis: float = field(init=False,repr=False,default=templates.EarthMinorAxis())
 650    resolutionAndComponentFlags: list = field(init=False,repr=False,default=templates.ResolutionAndComponentFlags())
 651    ny: int = field(init=False,repr=False,default=templates.Ny())
 652    nx: int = field(init=False,repr=False,default=templates.Nx())
 653    scanModeFlags: list = field(init=False,repr=False,default=templates.ScanModeFlags())
 654    projParameters: dict = field(init=False,repr=False,default=templates.ProjParameters())
 655
 656    # Section 4 attributes. Listed here are "extra" or "helper" attrs that use metadata from
 657    # the given PDT, but not a formal part of the PDT.
 658    productDefinitionTemplateNumber: Grib2Metadata = field(init=False,repr=False,default=templates.ProductDefinitionTemplateNumber())
 659    productDefinitionTemplate: np.array = field(init=False,repr=False,default=templates.ProductDefinitionTemplate())
 660    _varinfo: list = field(init=False, repr=False, default=templates.VarInfo())
 661    _fixedsfc1info: list = field(init=False, repr=False, default=templates.FixedSfc1Info())
 662    _fixedsfc2info: list = field(init=False, repr=False, default=templates.FixedSfc2Info())
 663    fullName: str = field(init=False, repr=False, default=templates.FullName())
 664    units: str = field(init=False, repr=False, default=templates.Units())
 665    shortName: str = field(init=False, repr=False, default=templates.ShortName())
 666    leadTime: datetime.timedelta = field(init=False,repr=False,default=templates.LeadTime())
 667    unitOfFirstFixedSurface: str = field(init=False,repr=False,default=templates.UnitOfFirstFixedSurface())
 668    valueOfFirstFixedSurface: int = field(init=False,repr=False,default=templates.ValueOfFirstFixedSurface())
 669    unitOfSecondFixedSurface: str = field(init=False,repr=False,default=templates.UnitOfSecondFixedSurface())
 670    valueOfSecondFixedSurface: int = field(init=False,repr=False,default=templates.ValueOfSecondFixedSurface())
 671    level: str = field(init=False, repr=False, default=templates.Level())
 672    duration: datetime.timedelta = field(init=False,repr=False,default=templates.Duration())
 673    validDate: datetime.datetime = field(init=False,repr=False,default=templates.ValidDate())
 674
 675    # Section 5 looked up common attributes.  Other looked up attributes are available according
 676    # to the Data Representation Template.
 677    numberOfPackedValues: int = field(init=False,repr=False,default=templates.NumberOfPackedValues())
 678    dataRepresentationTemplateNumber: Grib2Metadata = field(init=False,repr=False,default=templates.DataRepresentationTemplateNumber())
 679    dataRepresentationTemplate: list = field(init=False,repr=False,default=templates.DataRepresentationTemplate())
 680    typeOfValues: Grib2Metadata = field(init=False,repr=False,default=templates.TypeOfValues())
 681
 682
 683    def __post_init__(self):
 684        self._msgnum = -1
 685        self._deflist = None
 686        self._coordlist = None
 687        self._signature = self._generate_signature()
 688        try:
 689            self._sha1_section3 = hashlib.sha1(self.section3).hexdigest()
 690        except(TypeError):
 691            pass
 692        self.bitMapFlag = Grib2Metadata(self.bitMapFlag,table='6.0')
 693
 694
 695    @property
 696    def gdtn(self):
 697        return self.section3[4]
 698
 699
 700    @property
 701    def pdtn(self):
 702        return self.section4[1]
 703
 704
 705    @property
 706    def drtn(self):
 707        return self.section5[1]
 708
 709
 710    @property
 711    def pdy(self):
 712        return ''.join([str(i) for i in self.section1[5:8]])
 713
 714
 715    @property
 716    def griddef(self):
 717        return Grib2GridDef.from_section3(self.section3)
 718
 719
 720    def __repr__(self):
 721        info = ''
 722        for sect in [0,1,3,4,5,6]:
 723            for k,v in self.attrs_by_section(sect,values=True).items():
 724                info += f'Section {sect}: {k} = {v}\n'
 725        return info
 726
 727
 728    def __str__(self):
 729        return (f'{self._msgnum}:d={self.refDate}:{self.shortName}:'
 730                f'{self.fullName} ({self.units}):{self.level}:'
 731                f'{self.leadTime}')
 732
 733
 734    def _generate_signature(self):
 735        return hashlib.sha1(np.concatenate((self.section0,self.section1,
 736                                            self.section3,self.section4,
 737                                            self.section5))).hexdigest()
 738
 739
 740    def attrs_by_section(self, sect, values=False):
 741        """
 742        Provide a tuple of attribute names for the given GRIB2 section.
 743
 744        Parameters
 745        ----------
 746
 747        **`sect : int`**
 748
 749        The GRIB2 section number.
 750
 751        **`values : bool, optional`**
 752
 753        Optional (default is `False`) arugment to return attributes values.
 754
 755        Returns
 756        -------
 757
 758        A List attribute names or Dict if `values = True`.
 759        """
 760        if sect in {0,1,6}:
 761            attrs = templates._section_attrs[sect]
 762        elif sect in {3,4,5}:
 763            def _find_class_index(n):
 764                _key = {3:'Grid', 4:'Product', 5:'Data'}
 765                for i,c in enumerate(self.__class__.__mro__):
 766                    if _key[n] in c.__name__:
 767                        return i
 768                else:
 769                    return []
 770            if sys.version_info.minor <= 8:
 771                attrs = templates._section_attrs[sect]+\
 772                        [a for a in dir(self.__class__.__mro__[_find_class_index(sect)]) if not a.startswith('_')]
 773            else:
 774                attrs = templates._section_attrs[sect]+\
 775                        self.__class__.__mro__[_find_class_index(sect)]._attrs
 776        else:
 777            attrs = []
 778        if values:
 779            return {k:getattr(self,k) for k in attrs}
 780        else:
 781            return attrs
 782
 783
 784    def pack(self):
 785        """
 786        Packs GRIB2 section data into a binary message.  It is the user's responsibility
 787        to populate the GRIB2 section information with appropriate metadata.
 788        """
 789        # Create beginning of packed binary message with section 0 and 1 data.
 790        self._sections = []
 791        self._msg,self._pos = g2clib.grib2_create(self.indicatorSection[2:4],self.identificationSection)
 792        self._sections += [0,1]
 793
 794        # Add section 2 if present.
 795        if isinstance(self.section2,bytes) and len(self.section2) > 0:
 796            self._msg,self._pos = g2clib.grib2_addlocal(self._msg,self.section2)
 797            self._sections.append(2)
 798
 799        # Add section 3.
 800        self._msg,self._pos = g2clib.grib2_addgrid(self._msg,self.gridDefinitionSection,
 801                                                   self.gridDefinitionTemplate,
 802                                                   self._deflist)
 803        self._sections.append(3)
 804
 805        # Prepare data.
 806        field = np.copy(self.data)
 807        if self.scanModeFlags is not None:
 808            if self.scanModeFlags[3]:
 809                fieldsave = field.astype('f') # Casting makes a copy
 810                field[1::2,:] = fieldsave[1::2,::-1]
 811        fld = field.astype('f')
 812
 813        # Prepare bitmap, if necessary
 814        bitmapflag = self.bitMapFlag.value
 815        if bitmapflag == 0:
 816            bmap = np.ravel(np.where(np.isnan(fld),0,1)).astype(DEFAULT_NUMPY_INT)
 817        else:
 818            bmap = None
 819
 820        # Prepare optional coordinate list
 821        if self._coordlist is not None:
 822            crdlist = np.array(self._coordlist,'f')
 823        else:
 824            crdlist = None
 825
 826        # Add sections 4, 5, 6 (if present), and 7.
 827        self._msg,self._pos = g2clib.grib2_addfield(self._msg,self.pdtn,
 828                                                    self.productDefinitionTemplate,
 829                                                    crdlist,
 830                                                    self.drtn,
 831                                                    self.dataRepresentationTemplate,
 832                                                    np.ravel(fld),
 833                                                    bitmapflag,
 834                                                    bmap)
 835        self._sections.append(4)
 836        self._sections.append(5)
 837        if bmap is not None: self._sections.append(6)
 838        self._sections.append(7)
 839
 840        # Finalize GRIB2 message with section 8.
 841        self._msg, self._pos = g2clib.grib2_end(self._msg)
 842        self._sections.append(8)
 843        self.section0[-1] = len(self._msg)
 844
 845
 846    @property
 847    def data(self) -> np.array:
 848        """
 849        Accessing the data attribute loads data into memmory
 850        """
 851        if not hasattr(self,'_auto_nans'): self._auto_nans = _AUTO_NANS
 852        if hasattr(self,'_data'):
 853            if self._auto_nans != _AUTO_NANS:
 854                self._data = self._ondiskarray
 855            if isinstance(self._data, Grib2MessageOnDiskArray):
 856                self._ondiskarray = self._data
 857                self._data = np.asarray(self._data)
 858            return self._data
 859        raise ValueError
 860
 861    @data.setter
 862    def data(self, data):
 863        if not isinstance(data, np.ndarray):
 864            raise ValueError('Grib2Message data only supports numpy arrays')
 865        self._data = data
 866
 867
 868    def __getitem__(self, item):
 869        return self.data[item]
 870
 871
 872    def __setitem__(self, item):
 873        raise NotImplementedError('assignment of data not supported via setitem')
 874
 875
 876    def latlons(self, *args, **kwrgs):
 877        """Alias for `grib2io.Grib2Message.grid` method"""
 878        return self.grid(*args, **kwrgs)
 879
 880
 881    def grid(self, unrotate=True):
 882        """
 883        Return lats,lons (in degrees) of grid. Currently can handle reg. lat/lon,
 884        global Gaussian, mercator, stereographic, lambert conformal, albers equal-area,
 885        space-view and azimuthal equidistant grids.
 886
 887        Parameters
 888        ----------
 889
 890        **`unrotate : bool`**
 891
 892        If `True` [DEFAULT], and grid is rotated lat/lon, then unrotate the grid,
 893        otherwise `False`, do not.
 894
 895        Returns
 896        -------
 897
 898        **`lats, lons : numpy.ndarray`**
 899
 900        Returns two numpy.ndarrays with dtype=numpy.float32 of grid latitudes and
 901        longitudes in units of degrees.
 902        """
 903        if self._sha1_section3 in _latlon_datastore.keys():
 904            return _latlon_datastore[self._sha1_section3]
 905        gdtn = self.gridDefinitionTemplateNumber.value
 906        gdtmpl = self.gridDefinitionTemplate
 907        reggrid = self.gridDefinitionSection[2] == 0 # This means regular 2-d grid
 908        if gdtn == 0:
 909            # Regular lat/lon grid
 910            lon1, lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 911            lon2, lat2 = self.longitudeLastGridpoint, self.latitudeLastGridpoint
 912            dlon = self.gridlengthXDirection
 913            dlat = self.gridlengthYDirection
 914            if reggrid:
 915                lats = np.arange(lat1,lat2+dlat,dlat)
 916                lons = np.arange(lon1,lon2+dlon,dlon)
 917            else:
 918                lats = np.linspace(lat1,lat2,self.ny)
 919                lons = np.linspace(lon1,lon2,self.ny*2)
 920            lons,lats = np.meshgrid(lons,lats) # Make 2-d arrays.
 921        elif gdtn == 1: # Rotated Lat/Lon grid
 922            pj = pyproj.Proj(self.projParameters)
 923            lat1,lon1 = self.latitudeFirstGridpoint,self.longitudeFirstGridpoint
 924            lat2,lon2 = self.latitudeLastGridpoint,self.longitudeLastGridpoint
 925            if lon1 > 180.0: lon1 -= 360.0
 926            if lon2 > 180.0: lon2 -= 360.0
 927            lats = np.linspace(lat1,lat2,self.ny)
 928            lons = np.linspace(lon1,lon2,self.nx)
 929            lons,lats = np.meshgrid(lons,lats) # Make 2-d arrays.
 930            if unrotate:
 931                from grib2io.utils import rotated_grid
 932                lats,lons = rotated_grid.unrotate(lats,lons,self.anglePoleRotation,
 933                                                  self.latitudeSouthernPole,
 934                                                  self.longitudeSouthernPole)
 935        elif gdtn == 40: # Gaussian grid (only works for global!)
 936            from utils.gauss_grids import gaussian_latitudes
 937            lon1, lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 938            lon2, lat2 = self.longitudeLastGridpoint, self.latitudeLastGridpoint
 939            nlats = self.ny
 940            if not reggrid: # Reduced Gaussian grid.
 941                nlons = 2*nlats
 942                dlon = 360./nlons
 943            else:
 944                nlons = self.nx
 945                dlon = self.gridlengthXDirection
 946            lons = np.arange(lon1,lon2+dlon,dlon)
 947            # Compute Gaussian lats (north to south)
 948            lats = gaussian_latitudes(nlats)
 949            if lat1 < lat2:  # reverse them if necessary
 950                lats = lats[::-1]
 951            lons,lats = np.meshgrid(lons,lats)
 952        elif gdtn in {10,20,30,31,110}:
 953            # Mercator, Lambert Conformal, Stereographic, Albers Equal Area, Azimuthal Equidistant
 954            dx,dy = self.gridlengthXDirection, self.gridlengthYDirection
 955            lon1,lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 956            pj = pyproj.Proj(self.projParameters)
 957            llcrnrx, llcrnry = pj(lon1,lat1)
 958            x = llcrnrx+dx*np.arange(self.nx)
 959            y = llcrnry+dy*np.arange(self.ny)
 960            x,y = np.meshgrid(x, y)
 961            lons,lats = pj(x, y, inverse=True)
 962        elif gdtn == 90:
 963            # Satellite Projection
 964            dx = self.gridlengthXDirection
 965            dy = self.gridlengthYDirection
 966            pj = pyproj.Proj(self.projParameters)
 967            x = dx*np.indices((self.ny,self.nx),'f')[1,:,:]
 968            x -= 0.5*x.max()
 969            y = dy*np.indices((self.ny,self.nx),'f')[0,:,:]
 970            y -= 0.5*y.max()
 971            lons,lats = pj(x,y,inverse=True)
 972            # Set lons,lats to 1.e30 where undefined
 973            abslons = np.fabs(lons)
 974            abslats = np.fabs(lats)
 975            lons = np.where(abslons < 1.e20, lons, 1.e30)
 976            lats = np.where(abslats < 1.e20, lats, 1.e30)
 977        else:
 978            raise ValueError('Unsupported grid')
 979
 980        _latlon_datastore[self._sha1_section3] = (lats,lons)
 981
 982        return lats, lons
 983
 984
 985    def map_keys(self):
 986        """
 987        Returns an unpacked data grid where integer grid values are replaced with
 988        a string in which the numeric value is a representation of. These types
 989        of fields are cateogrical or classifications where data values do not
 990        represent an observable or predictable physical quantity.
 991
 992        An example of such a field field would be [Dominant Precipitation Type -
 993        DPTYPE](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_table4-201.shtml)
 994
 995        Returns
 996        -------
 997
 998        **`numpy.ndarray`** of string values per element.
 999        """
1000        hold_auto_nans = _AUTO_NANS
1001        set_auto_nans(False)
1002        if (np.all(self.section1[0:2]==[7,14]) and self.shortName == 'PWTHER') or \
1003        (np.all(self.section1[0:2]==[8,65535]) and self.shortName == 'WX'):
1004            keys = utils.decode_wx_strings(self.section2)
1005            if hasattr(self,'priMissingValue') and self.priMissingValue not in [None,0]:
1006                keys[int(self.priMissingValue)] = 'Missing'
1007            if hasattr(self,'secMissingValue') and self.secMissingValue not in [None,0]:
1008                keys[int(self.secMissingValue)] = 'Missing'
1009            u,inv = np.unique(self.data,return_inverse=True)
1010            fld = np.array([keys[x] for x in u])[inv].reshape(self.data.shape)
1011        else:
1012            # For data whose units are defined in a code table
1013            tbl = re.findall(r'\d\.\d+',self.units,re.IGNORECASE)[0]
1014            for k,v in tables.get_table(tbl).items():
1015                fld = np.where(fld==k,v,fld)
1016        set_auto_nans(hold_auto_nans)
1017        return fld
1018
1019
1020    def to_bytes(self, validate=True):
1021        """
1022        Return packed GRIB2 message in bytes format. This will be Useful for
1023        exporting data in non-file formats.  For example, can be used to
1024        output grib data directly to S3 using the boto3 client without the
1025        need to write a temporary file to upload first.
1026
1027        Parameters
1028        ----------
1029
1030        **`validate : bool, optional`**
1031
1032        If `True` (DEFAULT), validates first/last four bytes for proper
1033        formatting, else returns None. If `False`, message is output as is.
1034
1035        Returns
1036        -------
1037
1038        Returns GRIB2 formatted message as bytes.
1039        """
1040        if validate:
1041            if self._msg[0:4]+self._msg[-4:] == b'GRIB7777':
1042                return self._msg
1043            else:
1044                return None
1045        else:
1046            return self._msg
1047
1048
1049    def interpolate(self, method, grid_def_out, method_options=None):
1050        """
1051        Perform grid spatial interpolation via the [NCEPLIBS-ip library](https://github.com/NOAA-EMC/NCEPLIBS-ip).
1052
1053        Parameters
1054        ----------
1055
1056        **`method : int or str`**
1057
1058        Interpolate method to use. This can either be an integer or string using
1059        the following mapping:
1060
1061        | Interpolate Scheme | Integer Value |
1062        | :---:              | :---:         |
1063        | 'bilinear'         | 0             |
1064        | 'bicubic'          | 1             |
1065        | 'neighbor'         | 2             |
1066        | 'budget'           | 3             |
1067        | 'spectral'         | 4             |
1068        | 'neighbor-budget'  | 6             |
1069
1070        **`grid_def_out : grib2io.Grib2GridDef`**
1071
1072        Grib2GridDef object of the output grid.
1073
1074        **`method_options : list of ints, optional`**
1075
1076        Interpolation options. See the NCEPLIBS-ip doucmentation for
1077        more information on how these are used.
1078        """
1079        section0 = self.section0
1080        section0[-1] = 0
1081        gds = [0, grid_def_out.nx*grid_def_out.ny, 0, 255, grid_def_out.gdtn]
1082        section3 = np.concatenate((gds,grid_def_out.gdt))
1083
1084        msg = Grib2Message(section0,
1085                           self.section1,
1086                           self.section2,
1087                           section3,
1088                           self.section4,
1089                           self.section5,
1090                           self.bitMapFlag.value)
1091        msg._msgnum = -1
1092        msg._deflist = self._deflist
1093        msg._coordlist = self._coordlist
1094        shape = (msg.ny,msg.nx)
1095        ndim = 2
1096        if msg.typeOfValues == 0:
1097            dtype = 'float32'
1098        elif msg.typeOfValues == 1:
1099            dtype = 'int32'
1100        msg._data = interpolate(self.data,method,Grib2GridDef.from_section3(self.section3),grid_def_out,
1101                                method_options=method_options).reshape(msg.ny,msg.nx)
1102        return msg