60class open():
 61    """
 62    GRIB2 File Object.
 63
 64    A physical file can contain one or more GRIB2 messages.  When instantiated, class `grib2io.open`, 
 65    the file named `filename` is opened for reading (`mode = 'r'`) and is automatically indexed.  
 66    The indexing procedure reads some of the GRIB2 metadata for all GRIB2 Messages.  A GRIB2 Message 
 67    may contain submessages whereby Section 2-7 can be repeated.  grib2io accommodates for this by 
 68    flattening any GRIB2 submessages into multiple individual messages.
 69
 70    Attributes
 71    ----------
 72    **`mode : str`**
 73        File IO mode of opening the file.
 74
 75    **`name : str`**
 76        Full path name of the GRIB2 file.
 77
 78    **`messages : int`**
 79        Count of GRIB2 Messages contained in the file.
 80
 81    **`current_message : int`**
 82        Current position of the file in units of GRIB2 Messages.
 83
 84    **`size : int`**
 85        Size of the file in units of bytes.
 86
 87    **`closed : bool`**
 88        `True` is file handle is close; `False` otherwise.
 89
 90    **`variables : tuple`**
 91        Tuple containing a unique list of variable short names (i.e. GRIB2 abbreviation names).
 92
 93    **`levels : tuple`**
 94        Tuple containing a unique list of wgrib2-formatted level/layer strings.
 95    """
 96    __slots__ = ('_filehandle','_hasindex','_index','mode','name','messages',
 97                 'current_message','size','closed','variables','levels','_pos')
 98    def __init__(self, filename, mode='r', **kwargs):
 99        """
100        Initialize GRIB2 File object instance.
101
102        Parameters
103        ----------
104
105        **`filename : str`**
106            File name containing GRIB2 messages.
107
108        **`mode : str, optional`**
109            File access mode where `r` opens the files for reading only; `w` opens the file for writing.
110        """
111        if mode in {'a','r','w'}:
112            mode = mode+'b'
113            if 'w' in mode: mode += '+'
114            if 'a' in mode: mode += '+'
115        self._filehandle = builtins.open(filename,mode=mode,buffering=ONE_MB)
116        self._hasindex = False
117        self._index = {}
118        self.mode = mode
119        self.name = os.path.abspath(filename)
120        self.messages = 0
121        self.current_message = 0
122        self.size = os.path.getsize(self.name)
123        self.closed = self._filehandle.closed
124        self.levels = None
125        self.variables = None
126        if 'r' in self.mode:
127            try:
128                self._build_index(no_data=kwargs['_xarray_backend'])
129            except(KeyError):
130                self._build_index()
131        # FIX: Cannot perform reads on mode='a'
132        #if 'a' in self.mode and self.size > 0: self._build_index()
133
134
135    def __delete__(self, instance):
136        self.close()
137        del self._index
138
139
140    def __enter__(self):
141        return self
142
143
144    def __exit__(self, atype, value, traceback):
145        self.close()
146
147
148    def __iter__(self):
149        yield from self._index['msg']
150
151
152    def __len__(self):
153        return self.messages
154
155
156    def __repr__(self):
157        strings = []
158        for k in self.__slots__:
159            if k.startswith('_'): continue
160            strings.append('%s = %s\n'%(k,eval('self.'+k)))
161        return ''.join(strings)
162
163
164    def __getitem__(self, key):
165        if isinstance(key,int):
166            if abs(key) >= len(self._index['msg']):
167                raise IndexError("index out of range")
168            else:
169                return self._index['msg'][key]
170        elif isinstance(key,str):
171            return self.select(shortName=key)
172        elif isinstance(key,slice):
173            return self._index['msg'][key]
174        else:
175            raise KeyError('Key must be an integer, slice, or GRIB2 variable shortName.')
176
177
178    def _build_index(self, no_data=False):
179        """
180        Perform indexing of GRIB2 Messages.
181        """
182        # Initialize index dictionary
183        if not self._hasindex:
184            self._index['offset'] = []
185            self._index['bitmap_offset'] = []
186            self._index['data_offset'] = []
187            self._index['size'] = []
188            self._index['data_size'] = []
189            self._index['submessageOffset'] = []
190            self._index['submessageBeginSection'] = []
191            self._index['isSubmessage'] = []
192            self._index['messageNumber'] = []
193            self._index['msg'] = []
194            self._hasindex = True
195
196        # Iterate
197        while True:
198            try:
199                # Read first 4 bytes and decode...looking for "GRIB"
200                pos = self._filehandle.tell()
201                header = struct.unpack('>i',self._filehandle.read(4))[0]
202
203                # Test header. Then get information from GRIB2 Section 0: the discipline
204                # number, edition number (should always be 2), and GRIB2 message size.
205                # Then iterate to check for submessages.
206                if header.to_bytes(4,'big') == b'GRIB':
207
208                    _issubmessage = False
209                    _submsgoffset = 0
210                    _submsgbegin = 0
211                    _bmapflag = None
212
213                    # Read the rest of Section 0 using struct.
214                    section0 = np.concatenate(([header],list(struct.unpack('>HBBQ',self._filehandle.read(12)))),dtype=np.int64)
215                    assert section0[3] == 2
216
217                    # Read and unpack Section 1
218                    secsize = struct.unpack('>i',self._filehandle.read(4))[0]
219                    secnum = struct.unpack('>B',self._filehandle.read(1))[0]
220                    assert secnum == 1
221                    self._filehandle.seek(self._filehandle.tell()-5)
222                    _grbmsg = self._filehandle.read(secsize)
223                    _grbpos = 0
224                    section1,_grbpos = g2clib.unpack1(_grbmsg,_grbpos,np.empty)
225                    secrange = range(2,8)
226                    while 1:
227                        section2 = b''
228                        for num in secrange:
229                            secsize = struct.unpack('>i',self._filehandle.read(4))[0]
230                            secnum = struct.unpack('>B',self._filehandle.read(1))[0]
231                            if secnum == num:
232                                if secnum == 2:
233                                    if secsize > 0:
234                                        section2 = self._filehandle.read(secsize-5)
235                                elif secnum == 3:
236                                    self._filehandle.seek(self._filehandle.tell()-5)
237                                    _grbmsg = self._filehandle.read(secsize)
238                                    _grbpos = 0
239                                    # Unpack Section 3
240                                    _gds,_gdt,_deflist,_grbpos = g2clib.unpack3(_grbmsg,_grbpos,np.empty)
241                                    _gds = _gds.tolist()
242                                    _gdt = _gdt.tolist()
243                                    section3 = np.concatenate((_gds,_gdt))
244                                    section3 = np.where(section3==4294967295,-1,section3)
245                                elif secnum == 4:
246                                    self._filehandle.seek(self._filehandle.tell()-5)
247                                    _grbmsg = self._filehandle.read(secsize)
248                                    _grbpos = 0
249                                    # Unpack Section 4
250                                    _numcoord,_pdt,_pdtnum,_coordlist,_grbpos = g2clib.unpack4(_grbmsg,_grbpos,np.empty)
251                                    _pdt = _pdt.tolist()
252                                    section4 = np.concatenate((np.array((_numcoord,_pdtnum)),_pdt))
253                                elif secnum == 5:
254                                    self._filehandle.seek(self._filehandle.tell()-5)
255                                    _grbmsg = self._filehandle.read(secsize)
256                                    _grbpos = 0
257                                    # Unpack Section 5
258                                    _drt,_drtn,_npts,self._pos = g2clib.unpack5(_grbmsg,_grbpos,np.empty)
259                                    section5 = np.concatenate((np.array((_npts,_drtn)),_drt))
260                                    section5 = np.where(section5==4294967295,-1,section5)
261                                elif secnum == 6:
262                                    # Unpack Section 6. Not really...just get the flag value.
263                                    _bmapflag = struct.unpack('>B',self._filehandle.read(1))[0]
264                                    if _bmapflag == 0:
265                                        _bmappos = self._filehandle.tell()-6
266                                    elif _bmapflag == 254:
267                                        pass # Do this to keep the previous position value
268                                    else:
269                                        _bmappos = None
270                                    self._filehandle.seek(self._filehandle.tell()+secsize-6)
271                                elif secnum == 7:
272                                    # Unpack Section 7. No need to read it, just index the position in file.
273                                    _datapos = self._filehandle.tell()-5
274                                    _datasize = secsize
275                                    self._filehandle.seek(self._filehandle.tell()+secsize-5)
276                                else:
277                                    self._filehandle.seek(self._filehandle.tell()+secsize-5)
278                            else:
279                                if num == 2 and secnum == 3:
280                                    pass # Allow this.  Just means no Local Use Section.
281                                else:
282                                    _issubmessage = True
283                                    _submsgoffset = (self._filehandle.tell()-5)-(self._index['offset'][-1])
284                                    _submsgbegin = secnum
285                                self._filehandle.seek(self._filehandle.tell()-5)
286                                continue
287                        trailer = struct.unpack('>4s',self._filehandle.read(4))[0]
288                        if trailer == b'7777':
289                            self.messages += 1
290                            self._index['offset'].append(pos)
291                            self._index['bitmap_offset'].append(_bmappos)
292                            self._index['data_offset'].append(_datapos)
293                            self._index['size'].append(section0[-1])
294                            self._index['data_size'].append(_datasize)
295                            self._index['messageNumber'].append(self.messages)
296                            self._index['isSubmessage'].append(_issubmessage)
297                            if _issubmessage:
298                                self._index['submessageOffset'].append(_submsgoffset)
299                                self._index['submessageBeginSection'].append(_submsgbegin)
300                            else:
301                                self._index['submessageOffset'].append(0)
302                                self._index['submessageBeginSection'].append(_submsgbegin)
303
304                            # Create Grib2Message with data.
305                            msg = Grib2Message(section0,section1,section2,section3,section4,section5,_bmapflag)
306                            msg._msgnum = self.messages-1
307                            msg._deflist = _deflist
308                            msg._coordlist = _coordlist
309                            if not no_data:
310                                msg._data = Grib2MessageOnDiskArray((msg.ny,msg.nx), 2,
311                                                                    TYPE_OF_VALUES_DTYPE[msg.typeOfValues],
312                                                                    self._filehandle,
313                                                                    msg, pos, _bmappos, _datapos)
314                            self._index['msg'].append(msg)
315
316                            break
317                        else:
318                            self._filehandle.seek(self._filehandle.tell()-4)
319                            self.messages += 1
320                            self._index['offset'].append(pos)
321                            self._index['bitmap_offset'].append(_bmappos)
322                            self._index['data_offset'].append(_datapos)
323                            self._index['size'].append(section0[-1])
324                            self._index['data_size'].append(_datasize)
325                            self._index['messageNumber'].append(self.messages)
326                            self._index['isSubmessage'].append(_issubmessage)
327                            self._index['submessageOffset'].append(_submsgoffset)
328                            self._index['submessageBeginSection'].append(_submsgbegin)
329
330                            # Create Grib2Message with data.
331                            msg = Grib2Message(section0,section1,section2,section3,section4,section5,_bmapflag)
332                            msg._msgnum = self.messages-1
333                            msg._deflist = _deflist
334                            msg._coordlist = _coordlist
335                            if not no_data:
336                                msg._data = Grib2MessageOnDiskArray((msg.ny,msg.nx), 2,
337                                                                    TYPE_OF_VALUES_DTYPE[msg.typeOfValues],
338                                                                    self._filehandle,
339                                                                    msg, pos, _bmappos, _datapos)
340                            self._index['msg'].append(msg)
341
342                            continue
343
344            except(struct.error):
345                if 'r' in self.mode:
346                    self._filehandle.seek(0)
347                break
348
349        # Index at end of _build_index()
350        if self._hasindex and not no_data:
351             self.variables = tuple(sorted(set([msg.shortName for msg in self._index['msg']])))
352             self.levels = tuple(sorted(set([msg.level for msg in self._index['msg']])))
353
354
355    def close(self):
356        """
357        Close the file handle
358        """
359        if not self._filehandle.closed:
360            self.messages = 0
361            self.current_message = 0
362            self._filehandle.close()
363            self.closed = self._filehandle.closed
364
365
366    def read(self, size=None):
367        """
368        Read size amount of GRIB2 messages from the current position.
369
370        If no argument is given, then size is None and all messages are returned from 
371        the current position in the file. This read method follows the behavior of 
372        Python's builtin open() function, but whereas that operates on units of bytes, 
373        we operate on units of GRIB2 messages.
374
375        Parameters
376        ----------
377        **`size : int, optional`**
378            The number of GRIB2 messages to read from the current position. If no argument is
379            give, the default value is `None` and remainder of the file is read.
380
381        Returns
382        -------
383        `Grib2Message` object when size = 1 or a `list` of Grib2Messages when size > 1.
384        """
385        if size is not None and size < 0:
386            size = None
387        if size is None or size > 1:
388            start = self.tell()
389            stop = self.messages if size is None else start+size
390            if size is None:
391                self.current_message = self.messages-1
392            else:
393                self.current_message += size
394            return self._index['msg'][slice(start,stop,1)]
395        elif size == 1:
396            self.current_message += 1
397            return self._index['msg'][self.current_message]
398        else:
399            None
400
401
402    def seek(self, pos):
403        """
404        Set the position within the file in units of GRIB2 messages.
405
406        Parameters
407        ----------
408        **`pos : int`**
409            The GRIB2 Message number to set the file pointer to.
410        """
411        if self._hasindex:
412            self._filehandle.seek(self._index['offset'][pos])
413            self.current_message = pos
414
415
416    def tell(self):
417        """
418        Returns the position of the file in units of GRIB2 Messages.
419        """
420        return self.current_message
421
422
423    def select(self, **kwargs):
424        """
425        Select GRIB2 messages by `Grib2Message` attributes.
426        """
427        # TODO: Added ability to process multiple values for each keyword (attribute)
428        idxs = []
429        nkeys = len(kwargs.keys())
430        for k,v in kwargs.items():
431            for m in self._index['msg']:
432                if hasattr(m,k) and getattr(m,k) == v: idxs.append(m._msgnum)
433        idxs = np.array(idxs,dtype=np.int32)
434        return [self._index['msg'][i] for i in [ii[0] for ii in collections.Counter(idxs).most_common() if ii[1] == nkeys]]
435
436
437    def write(self, msg):
438        """
439        Writes GRIB2 message object to file.
440
441        Parameters
442        ----------
443        **`msg : Grib2Message or sequence of Grib2Messages`**
444            GRIB2 message objects to write to file.
445        """
446        if isinstance(msg,list):
447            for m in msg:
448                self.write(m)
449            return
450
451        if issubclass(msg.__class__,_Grib2Message):
452            if hasattr(msg,'_msg'):
453                self._filehandle.write(msg._msg)
454            else:
455                if msg._signature != msg._generate_signature():
456                    msg.pack()
457                    self._filehandle.write(msg._msg)
458                else:
459                    if hasattr(msg._data,'filehandle'):
460                        msg._data.filehandle.seek(msg._data.offset)
461                        self._filehandle.write(msg._data.filehandle.read(msg.section0[-1]))
462                    else:
463                        msg.pack()
464                        self._filehandle.write(msg._msg)
465            self.flush()
466            self.size = os.path.getsize(self.name)
467            self._filehandle.seek(self.size-msg.section0[-1])
468            self._build_index()
469        else:
470            raise TypeError("msg must be a Grib2Message object.")
471        return
472
473
474    def flush(self):
475        """
476        Flush the file object buffer.
477        """
478        self._filehandle.flush()
479
480
481    def levels_by_var(self, name):
482        """
483        Return a list of level strings given a variable shortName.
484
485        Parameters
486        ----------
487        **`name : str`**
488            Grib2Message variable shortName
489
490        Returns
491        -------
492        A list of strings of unique level strings.
493        """
494        return list(sorted(set([msg.level for msg in self.select(shortName=name)])))
495
496
497    def vars_by_level(self, level):
498        """
499        Return a list of variable shortName strings given a level.
500
501        Parameters
502        ----------
503        **`level : str`**
504            Grib2Message variable level
505
506        Returns
507        -------
508        A list of strings of variable shortName strings.
509        """
510        return list(sorted(set([msg.shortName for msg in self.select(level=level)])))

513class Grib2Message:
514    """
515    Creation class for a GRIB2 message.
516    """
517    def __new__(self, section0: np.array = np.array([struct.unpack('>I',b'GRIB')[0],0,0,2,0]),
518                      section1: np.array = np.zeros((13),dtype=np.int64),
519                      section2: bytes = None,
520                      section3: np.array = None,
521                      section4: np.array = None,
522                      section5: np.array = None, *args, **kwargs):
523
524        if np.all(section1==0):
525            section1[5:11] = datetime.datetime.utcfromtimestamp(0).timetuple()[:6]
526
527        bases = list()
528        if section3 is None:
529            if 'gdtn' in kwargs.keys():
530                gdtn = kwargs['gdtn']
531                Gdt = templates.gdt_class_by_gdtn(gdtn)
532                bases.append(Gdt)
533                section3 = np.zeros((Gdt._len+5),dtype=np.int64)
534                section3[4] = gdtn
535            else:
536                raise ValueError("Must provide GRIB2 Grid Definition Template Number or section 3 array")
537        else:
538            gdtn = section3[4]
539            Gdt = templates.gdt_class_by_gdtn(gdtn)
540            bases.append(Gdt)
541
542        if section4 is None:
543            if 'pdtn' in kwargs.keys():
544                pdtn = kwargs['pdtn']
545                Pdt = templates.pdt_class_by_pdtn(pdtn)
546                bases.append(Pdt)
547                section4 = np.zeros((Pdt._len+2),dtype=np.int64)
548                section4[1] = pdtn
549            else:
550                raise ValueError("Must provide GRIB2 Production Definition Template Number or section 4 array")
551        else:
552            pdtn = section4[1]
553            Pdt = templates.pdt_class_by_pdtn(pdtn)
554            bases.append(Pdt)
555
556        if section5 is None:
557            if 'drtn' in kwargs.keys():
558                drtn = kwargs['drtn']
559                Drt = templates.drt_class_by_drtn(drtn)
560                bases.append(Drt)
561                section5 = np.zeros((Drt._len+2),dtype=np.int64)
562                section5[1] = drtn
563            else:
564                raise ValueError("Must provide GRIB2 Data Representation Template Number or section 5 array")
565        else:
566            drtn = section5[1]
567            Drt = templates.drt_class_by_drtn(drtn)
568            bases.append(Drt)
569
570        # attempt to use existing Msg class if it has already been made with gdtn,pdtn,drtn combo
571        try:
572            Msg = _msg_class_store[f"{gdtn}:{pdtn}:{drtn}"]
573        except KeyError:
574            @dataclass(init=False, repr=False)
575            class Msg(_Grib2Message, *bases):
576                pass
577            _msg_class_store[f"{gdtn}:{pdtn}:{drtn}"] = Msg
578
579        return Msg(section0, section1, section2, section3, section4, section5, *args)

 582@dataclass
 583class _Grib2Message:
 584    """GRIB2 Message base class"""
 585    # GRIB2 Sections
 586    section0: np.array = field(init=True,repr=False)
 587    section1: np.array = field(init=True,repr=False)
 588    section2: bytes = field(init=True,repr=False)
 589    section3: np.array = field(init=True,repr=False)
 590    section4: np.array = field(init=True,repr=False)
 591    section5: np.array = field(init=True,repr=False)
 592    bitMapFlag: templates.Grib2Metadata = field(init=True,repr=False,default=255)
 593
 594    # Section 0 looked up attributes
 595    indicatorSection: np.array = field(init=False,repr=False,default=templates.IndicatorSection())
 596    discipline: templates.Grib2Metadata = field(init=False,repr=False,default=templates.Discipline())
 597
 598    # Section 1 looked up attributes
 599    identificationSection: np.array = field(init=False,repr=False,default=templates.IdentificationSection())
 600    originatingCenter: templates.Grib2Metadata = field(init=False,repr=False,default=templates.OriginatingCenter())
 601    originatingSubCenter: templates.Grib2Metadata = field(init=False,repr=False,default=templates.OriginatingSubCenter())
 602    masterTableInfo: templates.Grib2Metadata = field(init=False,repr=False,default=templates.MasterTableInfo())
 603    localTableInfo: templates.Grib2Metadata = field(init=False,repr=False,default=templates.LocalTableInfo())
 604    significanceOfReferenceTime: templates.Grib2Metadata = field(init=False,repr=False,default=templates.SignificanceOfReferenceTime())
 605    year: int = field(init=False,repr=False,default=templates.Year())
 606    month: int = field(init=False,repr=False,default=templates.Month())
 607    day: int = field(init=False,repr=False,default=templates.Day())
 608    hour: int = field(init=False,repr=False,default=templates.Hour())
 609    minute: int = field(init=False,repr=False,default=templates.Minute())
 610    second: int = field(init=False,repr=False,default=templates.Second())
 611    refDate: datetime.datetime = field(init=False,repr=False,default=templates.RefDate())
 612    productionStatus: templates.Grib2Metadata = field(init=False,repr=False,default=templates.ProductionStatus())
 613    typeOfData: templates.Grib2Metadata = field(init=False,repr=False,default=templates.TypeOfData())
 614
 615    @property
 616    def _isNDFD(self):
 617        """Check if GRIB2 message is from NWS NDFD"""
 618        return np.all(self.section1[0:2]==[8,65535])
 619
 620    # Section 3 looked up common attributes.  Other looked up attributes are available according
 621    # to the Grid Definition Template.
 622    gridDefinitionSection: np.array = field(init=False,repr=False,default=templates.GridDefinitionSection())
 623    sourceOfGridDefinition: int = field(init=False,repr=False,default=templates.SourceOfGridDefinition())
 624    numberOfDataPoints: int = field(init=False,repr=False,default=templates.NumberOfDataPoints())
 625    interpretationOfListOfNumbers: templates.Grib2Metadata = field(init=False,repr=False,default=templates.InterpretationOfListOfNumbers())
 626    gridDefinitionTemplateNumber: templates.Grib2Metadata = field(init=False,repr=False,default=templates.GridDefinitionTemplateNumber())
 627    gridDefinitionTemplate: list = field(init=False,repr=False,default=templates.GridDefinitionTemplate())
 628    _earthparams: dict = field(init=False,repr=False,default=templates.EarthParams())
 629    _dxsign: float = field(init=False,repr=False,default=templates.DxSign())
 630    _dysign: float = field(init=False,repr=False,default=templates.DySign())
 631    _llscalefactor: float = field(init=False,repr=False,default=templates.LLScaleFactor())
 632    _lldivisor: float = field(init=False,repr=False,default=templates.LLDivisor())
 633    _xydivisor: float = field(init=False,repr=False,default=templates.XYDivisor())
 634    shapeOfEarth: templates.Grib2Metadata = field(init=False,repr=False,default=templates.ShapeOfEarth())
 635    earthShape: str = field(init=False,repr=False,default=templates.EarthShape())
 636    earthRadius: float = field(init=False,repr=False,default=templates.EarthRadius())
 637    earthMajorAxis: float = field(init=False,repr=False,default=templates.EarthMajorAxis())
 638    earthMinorAxis: float = field(init=False,repr=False,default=templates.EarthMinorAxis())
 639    resolutionAndComponentFlags: list = field(init=False,repr=False,default=templates.ResolutionAndComponentFlags())
 640    ny: int = field(init=False,repr=False,default=templates.Ny())
 641    nx: int = field(init=False,repr=False,default=templates.Nx())
 642    scanModeFlags: list = field(init=False,repr=False,default=templates.ScanModeFlags())
 643    projParameters: dict = field(init=False,repr=False,default=templates.ProjParameters())
 644
 645    # Section 4 attributes. Listed here are "extra" or "helper" attrs that use metadata from
 646    # the given PDT, but not a formal part of the PDT.
 647    productDefinitionTemplateNumber: templates.Grib2Metadata = field(init=False,repr=False,default=templates.ProductDefinitionTemplateNumber())
 648    productDefinitionTemplate: np.array = field(init=False,repr=False,default=templates.ProductDefinitionTemplate())
 649    _varinfo: list = field(init=False, repr=False, default=templates.VarInfo())
 650    _fixedsfc1info: list = field(init=False, repr=False, default=templates.FixedSfc1Info())
 651    _fixedsfc2info: list = field(init=False, repr=False, default=templates.FixedSfc2Info())
 652    fullName: str = field(init=False, repr=False, default=templates.FullName())
 653    units: str = field(init=False, repr=False, default=templates.Units())
 654    shortName: str = field(init=False, repr=False, default=templates.ShortName())
 655    leadTime: datetime.timedelta = field(init=False,repr=False,default=templates.LeadTime())
 656    unitOfFirstFixedSurface: str = field(init=False,repr=False,default=templates.UnitOfFirstFixedSurface())
 657    valueOfFirstFixedSurface: int = field(init=False,repr=False,default=templates.ValueOfFirstFixedSurface())
 658    unitOfSecondFixedSurface: str = field(init=False,repr=False,default=templates.UnitOfSecondFixedSurface())
 659    valueOfSecondFixedSurface: int = field(init=False,repr=False,default=templates.ValueOfSecondFixedSurface())
 660    level: str = field(init=False, repr=False, default=templates.Level())
 661    duration: datetime.timedelta = field(init=False,repr=False,default=templates.Duration())
 662    validDate: datetime.datetime = field(init=False,repr=False,default=templates.ValidDate())
 663
 664    # Section 5 looked up common attributes.  Other looked up attributes are available according
 665    # to the Data Representation Template.
 666    numberOfPackedValues: int = field(init=False,repr=False,default=templates.NumberOfPackedValues())
 667    dataRepresentationTemplateNumber: templates.Grib2Metadata = field(init=False,repr=False,default=templates.DataRepresentationTemplateNumber())
 668    dataRepresentationTemplate: list = field(init=False,repr=False,default=templates.DataRepresentationTemplate())
 669    typeOfValues: templates.Grib2Metadata = field(init=False,repr=False,default=templates.TypeOfValues())
 670
 671
 672    def __post_init__(self):
 673        """Set some attributes after init"""
 674        self._msgnum = -1
 675        self._deflist = None
 676        self._coordlist = None
 677        self._signature = self._generate_signature()
 678        try:
 679            self._sha1_section3 = hashlib.sha1(self.section3).hexdigest()
 680        except(TypeError):
 681            pass
 682        self.bitMapFlag = templates.Grib2Metadata(self.bitMapFlag,table='6.0')
 683
 684
 685    @property
 686    def gdtn(self):
 687        """Return Grid Definition Template Number"""
 688        return self.section3[4]
 689
 690
 691    @property
 692    def gdt(self):
 693        """Return Grid Definition Template"""
 694        return self.gridDefinitionTemplate
 695
 696
 697    @property
 698    def pdtn(self):
 699        """Return Product Definition Template Number"""
 700        return self.section4[1]
 701
 702
 703    @property
 704    def pdt(self):
 705        """Return Product Definition Template"""
 706        return self.productDefinitionTemplate
 707
 708
 709    @property
 710    def drtn(self):
 711        """Return Data Representation Template Number"""
 712        return self.section5[1]
 713
 714
 715    @property
 716    def drt(self):
 717        """Return Data Representation Template"""
 718        return self.dataRepresentationTemplate
 719
 720
 721    @property
 722    def pdy(self):
 723        """Return the PDY ('YYYYMMDD')"""
 724        return ''.join([str(i) for i in self.section1[5:8]])
 725
 726
 727    @property
 728    def griddef(self):
 729        """Return a Grib2GridDef instance for a GRIB2 message"""
 730        return Grib2GridDef.from_section3(self.section3)
 731
 732
 733    def __repr__(self):
 734        info = ''
 735        for sect in [0,1,3,4,5,6]:
 736            for k,v in self.attrs_by_section(sect,values=True).items():
 737                info += f'Section {sect}: {k} = {v}\n'
 738        return info
 739
 740
 741    def __str__(self):
 742        return (f'{self._msgnum}:d={self.refDate}:{self.shortName}:'
 743                f'{self.fullName} ({self.units}):{self.level}:'
 744                f'{self.leadTime}')
 745
 746
 747    def _generate_signature(self):
 748        """Generature SHA-1 hash string from GRIB2 integer sections"""
 749        return hashlib.sha1(np.concatenate((self.section0,self.section1,
 750                                            self.section3,self.section4,
 751                                            self.section5))).hexdigest()
 752
 753
 754    def attrs_by_section(self, sect, values=False):
 755        """
 756        Provide a tuple of attribute names for the given GRIB2 section.
 757
 758        Parameters
 759        ----------
 760        **`sect : int`**
 761            The GRIB2 section number.
 762
 763        **`values : bool, optional`**
 764            Optional (default is `False`) arugment to return attributes values.
 765
 766        Returns
 767        -------
 768        A List attribute names or Dict if `values = True`.
 769        """
 770        if sect in {0,1,6}:
 771            attrs = templates._section_attrs[sect]
 772        elif sect in {3,4,5}:
 773            def _find_class_index(n):
 774                _key = {3:'Grid', 4:'Product', 5:'Data'}
 775                for i,c in enumerate(self.__class__.__mro__):
 776                    if _key[n] in c.__name__:
 777                        return i
 778                else:
 779                    return []
 780            if sys.version_info.minor <= 8:
 781                attrs = templates._section_attrs[sect]+\
 782                        [a for a in dir(self.__class__.__mro__[_find_class_index(sect)]) if not a.startswith('_')]
 783            else:
 784                attrs = templates._section_attrs[sect]+\
 785                        self.__class__.__mro__[_find_class_index(sect)]._attrs
 786        else:
 787            attrs = []
 788        if values:
 789            return {k:getattr(self,k) for k in attrs}
 790        else:
 791            return attrs
 792
 793
 794    def pack(self):
 795        """
 796        Packs GRIB2 section data into a binary message.  It is the user's responsibility
 797        to populate the GRIB2 section information with appropriate metadata.
 798        """
 799        # Create beginning of packed binary message with section 0 and 1 data.
 800        self._sections = []
 801        self._msg,self._pos = g2clib.grib2_create(self.indicatorSection[2:4],self.identificationSection)
 802        self._sections += [0,1]
 803
 804        # Add section 2 if present.
 805        if isinstance(self.section2,bytes) and len(self.section2) > 0:
 806            self._msg,self._pos = g2clib.grib2_addlocal(self._msg,self.section2)
 807            self._sections.append(2)
 808
 809        # Add section 3.
 810        self.section3[1] = self.nx * self.ny
 811        self._msg,self._pos = g2clib.grib2_addgrid(self._msg,self.gridDefinitionSection,
 812                                                   self.gridDefinitionTemplate,
 813                                                   self._deflist)
 814        self._sections.append(3)
 815
 816        # Prepare data.
 817        field = np.copy(self.data)
 818        if self.scanModeFlags is not None:
 819            if self.scanModeFlags[3]:
 820                fieldsave = field.astype('f') # Casting makes a copy
 821                field[1::2,:] = fieldsave[1::2,::-1]
 822        fld = field.astype('f')
 823
 824        # Prepare bitmap, if necessary
 825        bitmapflag = self.bitMapFlag.value
 826        if bitmapflag == 0:
 827            bmap = np.ravel(np.where(np.isnan(fld),0,1)).astype(DEFAULT_NUMPY_INT)
 828        else:
 829            bmap = None
 830
 831        # Prepare optional coordinate list
 832        if self._coordlist is not None:
 833            crdlist = np.array(self._coordlist,'f')
 834        else:
 835            crdlist = None
 836
 837        # Prepare data for packing if nans are present
 838        fld = np.ravel(fld)
 839        if np.isnan(fld).any() and hasattr(self,'_missvalmap'):
 840            fld = np.where(self._missvalmap==1,self.priMissingValue,fld)
 841            fld = np.where(self._missvalmap==2,self.secMissingValue,fld)
 842
 843        # Add sections 4, 5, 6 (if present), and 7.
 844        self._msg,self._pos = g2clib.grib2_addfield(self._msg,self.pdtn,
 845                                                    self.productDefinitionTemplate,
 846                                                    crdlist,
 847                                                    self.drtn,
 848                                                    self.dataRepresentationTemplate,
 849                                                    fld,
 850                                                    bitmapflag,
 851                                                    bmap)
 852        self._sections.append(4)
 853        self._sections.append(5)
 854        if bmap is not None: self._sections.append(6)
 855        self._sections.append(7)
 856
 857        # Finalize GRIB2 message with section 8.
 858        self._msg, self._pos = g2clib.grib2_end(self._msg)
 859        self._sections.append(8)
 860        self.section0[-1] = len(self._msg)
 861
 862
 863    @property
 864    def data(self) -> np.array:
 865        """
 866        Accessing the data attribute loads data into memmory
 867        """
 868        if not hasattr(self,'_auto_nans'): self._auto_nans = _AUTO_NANS
 869        if hasattr(self,'_data'):
 870            if self._auto_nans != _AUTO_NANS:
 871                self._data = self._ondiskarray
 872            if isinstance(self._data, Grib2MessageOnDiskArray):
 873                self._ondiskarray = self._data
 874                self._data = np.asarray(self._data)
 875            return self._data
 876        raise ValueError
 877
 878    @data.setter
 879    def data(self, data):
 880        if not isinstance(data, np.ndarray):
 881            raise ValueError('Grib2Message data only supports numpy arrays')
 882        self._data = data
 883
 884
 885    def __getitem__(self, item):
 886        return self.data[item]
 887
 888
 889    def __setitem__(self, item):
 890        raise NotImplementedError('assignment of data not supported via setitem')
 891
 892
 893    def latlons(self, *args, **kwrgs):
 894        """Alias for `grib2io.Grib2Message.grid` method"""
 895        return self.grid(*args, **kwrgs)
 896
 897
 898    def grid(self, unrotate=True):
 899        """
 900        Return lats,lons (in degrees) of grid.
 901
 902        Currently can handle reg. lat/lon,cglobal Gaussian, mercator, stereographic, 
 903        lambert conformal, albers equal-area, space-view and azimuthal equidistant grids.
 904
 905        Parameters
 906        ----------
 907        **`unrotate : bool`**
 908            If `True` [DEFAULT], and grid is rotated lat/lon, then unrotate the grid,
 909            otherwise `False`, do not.
 910
 911        Returns
 912        -------
 913        **`lats, lons : numpy.ndarray`**
 914            Returns two numpy.ndarrays with dtype=numpy.float32 of grid latitudes and
 915            longitudes in units of degrees.
 916        """
 917        if self._sha1_section3 in _latlon_datastore.keys():
 918            return (_latlon_datastore[self._sha1_section3]['latitude'],
 919                    _latlon_datastore[self._sha1_section3]['longitude'])
 920        else:
 921            _latlon_datastore[self._sha1_section3] = {}
 922        gdtn = self.gridDefinitionTemplateNumber.value
 923        gdtmpl = self.gridDefinitionTemplate
 924        reggrid = self.gridDefinitionSection[2] == 0 # This means regular 2-d grid
 925        if gdtn == 0:
 926            # Regular lat/lon grid
 927            lon1, lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 928            lon2, lat2 = self.longitudeLastGridpoint, self.latitudeLastGridpoint
 929            dlon = self.gridlengthXDirection
 930            dlat = self.gridlengthYDirection
 931            if lon2 < lon1 and dlon < 0: lon1 = -lon1
 932            lats = np.linspace(lat1,lat2,self.ny)
 933            if reggrid:
 934                lons = np.linspace(lon1,lon2,self.nx)
 935            else:
 936                lons = np.linspace(lon1,lon2,self.ny*2)
 937            lons,lats = np.meshgrid(lons,lats) # Make 2-d arrays.
 938        elif gdtn == 1: # Rotated Lat/Lon grid
 939            pj = pyproj.Proj(self.projParameters)
 940            lat1,lon1 = self.latitudeFirstGridpoint,self.longitudeFirstGridpoint
 941            lat2,lon2 = self.latitudeLastGridpoint,self.longitudeLastGridpoint
 942            if lon1 > 180.0: lon1 -= 360.0
 943            if lon2 > 180.0: lon2 -= 360.0
 944            lats = np.linspace(lat1,lat2,self.ny)
 945            lons = np.linspace(lon1,lon2,self.nx)
 946            lons,lats = np.meshgrid(lons,lats) # Make 2-d arrays.
 947            if unrotate:
 948                from grib2io.utils import rotated_grid
 949                lats,lons = rotated_grid.unrotate(lats,lons,self.anglePoleRotation,
 950                                                  self.latitudeSouthernPole,
 951                                                  self.longitudeSouthernPole)
 952        elif gdtn == 40: # Gaussian grid (only works for global!)
 953            from utils.gauss_grids import gaussian_latitudes
 954            lon1, lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 955            lon2, lat2 = self.longitudeLastGridpoint, self.latitudeLastGridpoint
 956            nlats = self.ny
 957            if not reggrid: # Reduced Gaussian grid.
 958                nlons = 2*nlats
 959                dlon = 360./nlons
 960            else:
 961                nlons = self.nx
 962                dlon = self.gridlengthXDirection
 963            lons = np.arange(lon1,lon2+dlon,dlon)
 964            # Compute Gaussian lats (north to south)
 965            lats = gaussian_latitudes(nlats)
 966            if lat1 < lat2:  # reverse them if necessary
 967                lats = lats[::-1]
 968            lons,lats = np.meshgrid(lons,lats)
 969        elif gdtn in {10,20,30,31,110}:
 970            # Mercator, Lambert Conformal, Stereographic, Albers Equal Area, Azimuthal Equidistant
 971            dx,dy = self.gridlengthXDirection, self.gridlengthYDirection
 972            lon1,lat1 = self.longitudeFirstGridpoint, self.latitudeFirstGridpoint
 973            pj = pyproj.Proj(self.projParameters)
 974            llcrnrx, llcrnry = pj(lon1,lat1)
 975            x = llcrnrx+dx*np.arange(self.nx)
 976            y = llcrnry+dy*np.arange(self.ny)
 977            x,y = np.meshgrid(x, y)
 978            lons,lats = pj(x, y, inverse=True)
 979        elif gdtn == 90:
 980            # Satellite Projection
 981            dx = self.gridlengthXDirection
 982            dy = self.gridlengthYDirection
 983            pj = pyproj.Proj(self.projParameters)
 984            x = dx*np.indices((self.ny,self.nx),'f')[1,:,:]
 985            x -= 0.5*x.max()
 986            y = dy*np.indices((self.ny,self.nx),'f')[0,:,:]
 987            y -= 0.5*y.max()
 988            lons,lats = pj(x,y,inverse=True)
 989            # Set lons,lats to 1.e30 where undefined
 990            abslons = np.fabs(lons)
 991            abslats = np.fabs(lats)
 992            lons = np.where(abslons < 1.e20, lons, 1.e30)
 993            lats = np.where(abslats < 1.e20, lats, 1.e30)
 994        elif gdtn == 32769:
 995            # Special NCEP Grid, Rotated Lat/Lon, Arakawa E-Grid (Non-Staggered)
 996            from grib2io.utils import arakawa_rotated_grid
 997            from grib2io.utils.rotated_grid import DEG2RAD
 998            di, dj = 0.0, 0.0
 999            do_180 = False
1000            idir = 1 if self.scanModeFlags[0] == 0 else -1
1001            jdir = -1 if self.scanModeFlags[1] == 0 else 1
1002            grid_oriented = 0 if self.resolutionAndComponentFlags[4] == 0 else 1
1003            do_rot = 1 if grid_oriented == 1 else 0
1004            la1 = self.latitudeFirstGridpoint
1005            lo1 = self.longitudeFirstGridpoint
1006            clon = self.longitudeCenterGridpoint
1007            clat = self.latitudeCenterGridpoint
1008            lasp = clat - 90.0
1009            losp = clon
1010            llat, llon = arakawa_rotated_grid.ll2rot(la1,lo1,lasp,losp)
1011            la2, lo2 = arakawa_rotated_grid.rot2ll(-llat,-llon,lasp,losp)
1012            rlat = -llat
1013            rlon = -llon
1014            if self.nx == 1:
1015                di = 0.0
1016            elif idir == 1:
1017                ti = rlon
1018                while ti < llon:
1019                    ti += 360.0
1020                di = (ti - llon)/float(self.nx-1)
1021            else:
1022                ti = llon
1023                while ti < rlon:
1024                    ti += 360.0
1025                di = (ti - rlon)/float(self.nx-1)
1026            if self.ny == 1:
1027               dj = 0.0
1028            else:
1029                dj = (rlat - llat)/float(self.ny-1)
1030                if dj < 0.0:
1031                    dj = -dj
1032            if idir == 1:
1033                if llon > rlon:
1034                    llon -= 360.0
1035                if llon < 0 and rlon > 0:
1036                    do_180 = True
1037            else:
1038                if rlon > llon:
1039                    rlon -= 360.0
1040                if rlon < 0 and llon > 0:
1041                    do_180 = True
1042            xlat1d = llat + (np.arange(self.ny)*jdir*dj)
1043            xlon1d = llon + (np.arange(self.nx)*idir*di)
1044            xlons, xlats = np.meshgrid(xlon1d,xlat1d)
1045            rot2ll_vectorized = np.vectorize(arakawa_rotated_grid.rot2ll)
1046            lats, lons = rot2ll_vectorized(xlats,xlons,lasp,losp)
1047            if do_180:
1048                lons = np.where(lons>180.0,lons-360.0,lons)
1049            vector_rotation_angles_vectorized = np.vectorize(arakawa_rotated_grid.vector_rotation_angles)
1050            rots = vector_rotation_angles_vectorized(lats, lons, clat, losp, xlats)
1051            _latlon_datastore[self._sha1_section3]['vector_rotation_angles'] = rots
1052            del xlat1d, xlon1d, xlats, xlons
1053        else:
1054            raise ValueError('Unsupported grid')
1055
1056        _latlon_datastore[self._sha1_section3]['latitude'] = lats
1057        _latlon_datastore[self._sha1_section3]['longitude'] = lons
1058
1059        return lats, lons
1060
1061
1062    def map_keys(self):
1063        """
1064        Returns an unpacked data grid where integer grid values are replaced with
1065        a string.in which the numeric value is a representation of.
1066
1067        These types of fields are cateogrical or classifications where data values 
1068        do not represent an observable or predictable physical quantity. An example 
1069        of such a field field would be [Dominant Precipitation Type -
1070        DPTYPE](https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_table4-201.shtml)
1071
1072        Returns
1073        -------
1074        **`numpy.ndarray`** of string values per element.
1075        """
1076        hold_auto_nans = _AUTO_NANS
1077        set_auto_nans(False)
1078        if (np.all(self.section1[0:2]==[7,14]) and self.shortName == 'PWTHER') or \
1079        (np.all(self.section1[0:2]==[8,65535]) and self.shortName == 'WX'):
1080            keys = utils.decode_wx_strings(self.section2)
1081            if hasattr(self,'priMissingValue') and self.priMissingValue not in [None,0]:
1082                keys[int(self.priMissingValue)] = 'Missing'
1083            if hasattr(self,'secMissingValue') and self.secMissingValue not in [None,0]:
1084                keys[int(self.secMissingValue)] = 'Missing'
1085            u,inv = np.unique(self.data,return_inverse=True)
1086            fld = np.array([keys[x] for x in u])[inv].reshape(self.data.shape)
1087        else:
1088            # For data whose units are defined in a code table (i.e. classification or mask)
1089            tblname = re.findall(r'\d\.\d+',self.units,re.IGNORECASE)[0]
1090            fld = self.data.astype(np.int32).astype(str)
1091            tbl = tables.get_table(tblname,expand=True)
1092            for val in np.unique(fld):
1093                fld = np.where(fld==val,tbl[val],fld)
1094        set_auto_nans(hold_auto_nans)
1095        return fld
1096
1097
1098    def to_bytes(self, validate=True):
1099        """
1100        Return packed GRIB2 message in bytes format.
1101
1102        This will be Useful for exporting data in non-file formats. For example, 
1103        can be used to output grib data directly to S3 using the boto3 client 
1104        without the need to write a temporary file to upload first.
1105
1106        Parameters
1107        ----------
1108        **`validate : bool, optional`**
1109            If `True` (DEFAULT), validates first/last four bytes for proper
1110            formatting, else returns None. If `False`, message is output as is.
1111
1112        Returns
1113        -------
1114        Returns GRIB2 formatted message as bytes.
1115        """
1116        if validate:
1117            if self._msg[0:4]+self._msg[-4:] == b'GRIB7777':
1118                return self._msg
1119            else:
1120                return None
1121        else:
1122            return self._msg
1123
1124
1125    def interpolate(self, method, grid_def_out, method_options=None):
1126        """
1127        Perform grid spatial interpolation via the [NCEPLIBS-ip library](https://github.com/NOAA-EMC/NCEPLIBS-ip).
1128
1129        **IMPORTANT:**  This interpolate method only supports scalar interpolation. If you
1130        need to perform vector interpolation, use the module-level `grib2io.interpolate` function.
1131
1132        Parameters
1133        ----------
1134        **`method : int or str`**
1135            Interpolate method to use. This can either be an integer or string using
1136            the following mapping:
1137
1138        | Interpolate Scheme | Integer Value |
1139        | :---:              | :---:         |
1140        | 'bilinear'         | 0             |
1141        | 'bicubic'          | 1             |
1142        | 'neighbor'         | 2             |
1143        | 'budget'           | 3             |
1144        | 'spectral'         | 4             |
1145        | 'neighbor-budget'  | 6             |
1146
1147        **`grid_def_out : grib2io.Grib2GridDef`**
1148            Grib2GridDef object of the output grid.
1149
1150        **`method_options : list of ints, optional`**
1151            Interpolation options. See the NCEPLIBS-ip doucmentation for
1152            more information on how these are used.
1153
1154        Returns
1155        -------
1156        If interpolating to a grid, a new Grib2Message object is returned.  The GRIB2 metadata of
1157        the new Grib2Message object is indentical to the input except where required to be different
1158        because of the new grid specs.
1159
1160        If interpolating to station points, the interpolated data values are returned as a numpy.ndarray.
1161        """
1162        section0 = self.section0
1163        section0[-1] = 0
1164        gds = [0, grid_def_out.npoints, 0, 255, grid_def_out.gdtn]
1165        section3 = np.concatenate((gds,grid_def_out.gdt))
1166
1167        msg = Grib2Message(section0,self.section1,self.section2,section3,
1168                           self.section4,self.section5,self.bitMapFlag.value)
1169
1170        msg._msgnum = -1
1171        msg._deflist = self._deflist
1172        msg._coordlist = self._coordlist
1173        shape = (msg.ny,msg.nx)
1174        ndim = 2
1175        if msg.typeOfValues == 0:
1176            dtype = 'float32'
1177        elif msg.typeOfValues == 1:
1178            dtype = 'int32'
1179        msg._data = interpolate(self.data,method,Grib2GridDef.from_section3(self.section3),grid_def_out,
1180                                method_options=method_options).reshape(msg.ny,msg.nx)
1181        return msg