#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
#Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved.

#This program is free software; you can redistribute it and/or modify
#it under the terms of the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.

#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#Apache License, Version 2.0 for more details.


import re
import sys
import hpproto
import time
import shutil
import proto_writer
from optparse import OptionParser
import pyang
from pyang.error import Position
from hpproto.proto_proto import MessageObj
from hpproto.proto_proto import AnnotationObj
from hpproto.proto_proto import ProtoFile
import yang_util
from merge_proto_module import *

rep_revs_log_file = 'convert_yang2proto'
yang_root_node_keyword = 'container'
yang_top_node_keywords = ['notification', 'anydata', 'anyxml', 'container']
enum_type = ['bit', 'enumeration']
message_type = ['anydata', 'anyxml', 'augment', 'case', 'container', 'notification']
repeated_type = ['leaf-list', 'list']
oneof_type = ['choice']
yang_keyword_not_support = ['action']
proto_version = 'proto3'
yangtype_to_prototype = {
    'binary': 'bytes',
    'bits': 'enum',
    'boolean': 'bool',
    'empty': 'bool',
    'enumeration': 'enum',
    'instance-identifier': 'string',
    'int8': 'int32',
    'int16': 'int32',
    'int32': 'int32',
    'int64': 'int64',
    'string': 'string',
    'uint8': 'uint32',
    'uint16': 'uint32',
    'uint32': 'uint32',
    'uint64': 'uint64',
    'union': 'string',
    'identityref': 'string',
    'decimal64': 'Decimal64'
}
enum_val_format = ["!", '"', "#", "￥", "$", "%", "&", ",", "-", "(", ")", "*", "+", "/", ":", "<", ">", "=", ".",
                   "?", "[", "]", "{", "}", "|", "^", "~", "`", "@", " ", ";"]
name_not_support = []
same_name_suffix = '_1'
number = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '_']
origin_keep_word = ["BUILDPARTIAL", "CLEAR", "CLEARFIELD", "CLEARONEOF", "CLONE", "EQUALS", "FINDVALUEBYNUMBER",
                    "FORNUMBER", "GETDEFAULTINSTANCE", "GETDEFAULTINSTANCEFORTYPE", "GETDESCRIPTOR",
                    "GETDESCRIPTORFORTYPE", "GETNUMBER", "GETPARSERFORTYPE", "GETSERIALIZEDSIZE", "GETUNKNOWNFIELDS",
                    "GETVALUEDESCRIPTOR", "HASHCODE", "INTERNALGETFIELDACCESSORTABLE", "INTERNALGETVALUEMAP",
                    "INTERNALVALUEMAP", "ISINITIALIZED", "MAYBEFORCEBUILDERINITIALIZATION", "MERGEFROM",
                    "MERGEUNKNOWNFIELDS", "NEWBUILDER", "NEWBUILDERFORTYPE", "PARSEDELIMITEDFROM", "PARSEFROM",
                    "PARSEPARTIALFROM", "PARSER", "REGISTERALLEXTENSIONS", "SETFIELD", "SETREPEATEDFIELD",
                    "SETUNKNOWNFIELDS", "TOBUILDER", "VALUEOF", "WRITETO"]
ext_keep_word = ["DEFAULTINSTANCE", "DEFAULTINSTANCEFORTYPE", "DESCRIPTOR", "DESCRIPTORFORTYPE", "NUMBER",
                     "PARSERFORTYPE", "SERIALIZEDSIZE", "UNKNOWNFIELDS", "VALUEDESCRIPTOR", "FIELD", "REPEATEDFIELD",
                     "UNKNOWNFIELDS"]
# space not allowed in yang rfc
compile_str = '// compiled %s\n' % time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
annotations = [
    '// Automatically generated by ProtoYang\n',
    compile_str, '// ProtoYang-version 1\n',
]


def env_parse():
    parser = OptionParser()
    parser.add_option("-Y", "--yang", dest="yang_path", default=r'./yang',
                      help="The directory of yang files or a single yang file.",
                      metavar='"COMMON YANG FILE"')
    parser.add_option("-O", "--out", dest="out_proto_directory", default=r'./proto',
                      help="The directory for generated .proto files",
                      metavar='"ADD FILE DIRECTORY"')
    parser.add_option("-S", "--src", dest="src_proto_directory", default=r'./src_proto',
                      help="The source .proto file directory",
                      metavar='"SOURCE PROTO FILE DIRECTORY"')
    parser.add_option("-L", "--logdir", dest="log_directory", default=r'./logs',
                      help="The direcotry to create the log file",
                      metavar='"LOG DIRECTORY"')
    parser.add_option("-D", "--dependencies", dest="yang_dependencies", default=r'./yang_dependencies',
                      help="The directory used as yang dependencies ",
                      metavar='"DEPENDENCIES DIR"')
    (options, args) = parser.parse_args()
    # for directory in options.yang_path, \
    #                  options.out_proto_directory, \
    #                  options.log_directory:
    #     if not directory or not os.path.isdir(directory):
    #         sys.stderr.write("[0x000001] :directory doesn't exists: %s\n" % directory)
    #         sys.exit(1)
    if not os.path.exists(options.yang_path):
        sys.stderr.write("[0x000001] :yang path doesn't exists: %s\n" % options.yang_path)
        sys.exit(1)
    return options

class Yang_proto_converter():
    def __init__(self, options):
        self.yang_path = options.yang_path
        self.out_proto_directory = options.out_proto_directory
        self.log_directory = options.log_directory
        self.src_proto_directory = options.src_proto_directory
        self.yang_dependencies = options.yang_dependencies
    def translate_services(self):
        return

    def split_same_name_inonestruct(self, structs):
        names = []
        stat_list = structs.statement
        for stat in stat_list[:]:
            if stat.type is not None:
                if type(stat.type) is not str:
                    if stat.name == stat.type.name:
                        stat.type.name = ''.join([stat.name, same_name_suffix])
                    self.split_same_name_inonestruct(stat.type)
                else:
                    if stat.name == stat.type:
                        stat.name = ''.join([stat.name, same_name_suffix])
            if stat.name in names:
                # 当enum中出现重复时，直接删除重复的enum，不进行重命名
                # 遇到有重复的enum，按yang文件顺序从前往后删除重复的enum
                if structs.keyword == 'enum':
                    logging.info('delete duplicate enum:%s, xpath:%s' % (stat.name, get_proto_xpaths(stat)))
                    stat_list.remove(stat)
                    continue
                else:
                    stat.name = ''.join([stat.name, same_name_suffix])

            names.append(stat.name)

    def add_decimal64_definion(self, proto_file_name, proto_node, proto_file):
        if self.check_proto_has_decimal64(proto_node):
            self.translate_decimal64_to_proto(proto_file_name, proto_file)

    def check_proto_has_decimal64(self, proto_node):
        if not proto_node:
            return False
        for def_obj in proto_node.values():
            if self.check_proto_has_decimal64(def_obj.definions):
                return True
            for stat in def_obj.statement:
                if stat.type == 'Decimal64':
                    return True
                elif isinstance(stat.type, DefinionObj):
                    if self.check_proto_has_decimal64(stat.type.definions):
                        return True
        return False

    def _do_find_augment_node_parent(self, nodes, pos, stat):
        if pos == 0:
            return stat

        if stat.parent.arg == nodes[pos - 1]:
            stat = stat.parent
            pos -= 1
            return self._do_find_augment_node_parent(nodes, pos, stat)

    def find_augment_node_parent(self, xpath, nodes, yang_node):
        for stat in yang_node.substmts:
            if stat.arg == xpath and stat.i_target_node is not None:
                return self._do_find_augment_node_parent(nodes, len(nodes) - 1, stat.i_target_node)

    def get_module_full_name(self, module_prefix, yang_node):
        return yang_node.i_prefixes[module_prefix][0]

    def translate_augment(self, yang_node, ptx):
        use_augment = False
        augment_path = []
        module_prefixs = []
        nodes_str = []
        for stat in yang_node.substmts:
            if stat.keyword == 'augment':
                augment_path.append(stat.arg)
                use_augment = True

        if use_augment is not True:
            return

        augmented_module_prefix_list = []
        for xpath in augment_path:
            module_prefixs.clear()
            nodes_str.clear()

            logging.info('augment: current module:%s, xpath:%s', yang_node.i_modulename, xpath)
            pattern_module = re.compile(r'/[\w\-]*:[\w\-]*')
            m = pattern_module.findall(xpath)
            if m:
                for match_path in m:
                    module_prefix_str = match_path.split(':')[0].strip('/')
                    node_str = match_path.split(':')[1]
                    module_prefixs.append(module_prefix_str)
                    nodes_str.append(node_str)
            else:
                logging.info('xpath no match:%s', xpath)
                continue

            augmented_module_prefix = module_prefixs[0]
            if augmented_module_prefix not in augmented_module_prefix_list:
                augmented_module_prefix_list.append(augmented_module_prefix)
            else:
                continue

            parent_node = self.find_augment_node_parent(xpath, nodes_str, yang_node)
            if parent_node is None:
                logging.error('xpath(%s) not found parent node.' % xpath)
                continue

            # 将augment的模块写成aug临时文件
            # 可能存在augment多个原始yang文件情形，比如a.yang同时对b.yang和c.yang进行augment
            # augment临时文件命名：augment module name+ '-' + '原始modulename' + '_aug.proto'
            top_parent_node = parent_node.parent
            if top_parent_node is not None:
                proto_file_name = os.path.join(self.out_directory, '%s-%s_aug.proto'
                                               % (yang_node.i_modulename,top_parent_node.i_modulename))
                proto_file = ProtoFile(proto_file_name, proto_version)
                self.translate_heads(proto_file_name, top_parent_node, proto_file)
                self.translate_services()
                self.translate_messages(proto_file_name, top_parent_node, proto_file)
                old_proto_file_name = os.path.join(self.src_proto_directory, '%s.proto'% top_parent_node.i_modulename)
                m = self.get_module_for_out(old_proto_file_name, ptx, top_parent_node.i_modulename, proto_file)
                if ptx.errors:
                    return
                if m:
                    m.output_proto_file(proto_file_name)
                logging.info('generate tmp augment proto file(%s)', proto_file_name)
        return

    def find_top_yang_node_order_num(self, node, top_nodes):
        i = 0
        top_nodes_len = len(top_nodes)
        for i in range(0, top_nodes_len):
            if top_nodes[i].arg == node.arg:
                break

        return i

    def translate_messages(self, proto_file_name, yang_node, proto_file):
        top_nodes = yang_node.i_children
        for node in top_nodes:
            if node.keyword not in yang_top_node_keywords:
                continue
            proto_module = DefinionObj(proto_file_name)

            proto_module.keyword = 'message'
            name = self.get_out_message_definion_name(node.arg, '')
            proto_module.name = name
            proto_module.proto_xpath = '/%s' % name
            proto_module = self._parse_statement(proto_file_name, proto_module, node)

            self.split_same_name_inonestruct(proto_module)
            proto_file.definions[name] = proto_module
            if node.keyword == yang_root_node_keyword:
                proto_file.i_root_elements.append(name)

            proto_module.yang_node_order_num = self.find_top_yang_node_order_num(node, top_nodes)
        self.add_decimal64_definion(proto_file_name, proto_file.definions, proto_file)

    def get_state_from_module(self, yang_module):
        if yang_module.keyword in message_type:
            return 'message'
        if yang_module.keyword in repeated_type:
            return'repeated'
        if yang_module.keyword in oneof_type:
            return 'oneof'
        else:
            return 'simple'

    def first_letter_upper(self, name):
        # 当definion为'_24GHz'时，split('_')后传入的name为空，会产生异常
        strs = ''
        try:
            strs = ''.join([name[0].upper(), name[1:]])
        except:
            pass

        return strs

    def first_attr_upper(self, name):
        if name[0].isupper():
            return True
        return False

    def get_out_message_definion_name(self, name, father_proto_xpath):
        if '_' in name:
            for attr in name.split('_'):
                if not attr.istitle():
                    name = name.replace('_%s' % attr, self.first_letter_upper(attr))
        if '-' in name:
            for attr in name.split('-'):
                if not self.first_attr_upper(attr):
                    name = name.replace('-%s' % attr, self.first_letter_upper(attr))
                else:
                    name = name.replace('-%s' % attr, '_%s'% self.first_letter_upper(attr))
        index = 0
        for str in name:
            index += 1
            if str in number and index < len(name):
                name = name.replace(''.join([str, name[index]]), ''.join([str, name[index].upper()]))
        name = self.first_letter_upper(name)
        # calculate if same node occured
        name = self.calculate_proto_name(father_proto_xpath, name)
        return name

    def get_enumval_type(self, node_type, enum_val, father_proto_xpath):
        for enum_spec in enum_val_format:
            if enum_spec in enum_val:
                enum_val = enum_val.replace(enum_spec, 'X_')

        for enum_spec in number:
            if enum_val.startswith(enum_spec):
                enum_val = 'X_%s' % enum_val
                break
        if '-' in enum_val:
            enum_val = enum_val.replace('-', '_')
        enum_name = self.calculate_proto_name(father_proto_xpath, '%s_%s' % (node_type, enum_val.upper()))
        return enum_name

    def get_type_name_for_leaf(self, yang_module):
        try:
            type_module = yang_module.search_one('type')
            if type_module.i_type_spec is None:
                logging.error('type %s in %s line %s can not be find', type_module.arg,
                              os.path.basename(type_module.pos.ref), type_module.pos.line)
                return ''
            if yang_module.i_leafref is None:
                return yangtype_to_prototype[type_module.i_type_spec.name]
            else:
                return self.get_type_name_for_leaf(type_module.i_type_spec.i_target_node)
        except Exception:
            logging.error('get_type_name_for_leaf %s in module %s' % (yang_module.arg, self.module_name))

    def split_yang_node(self, node):
        results_node = {}
        results_node['message'] = []
        results_node['repeated'] = []
        results_node['oneof'] = []
        results_node['simple'] = []
        try:
            for i_node in node.i_children:
                results_node[self.get_state_from_module(i_node)].append(i_node)
        except:
            pass
        return results_node

    def _get_enum_value_exec(self, enum_name, enum_statement):
        enum_val = -1
        for stat in enum_statement.substmts:
            if stat.arg == enum_name:
                for enum_body in stat.substmts:
                    if enum_body.keyword == 'value':
                        enum_val = int(enum_body.arg)
                        break
                break
        return enum_val

    # 获取yang文件中enum的value，没有value时，赋值-1
    # 在enum name存在重复时，删除后，可以依据enum value排序
    def get_enum_value(self, enum_name, type_module):
        enum_val = -1
        if type_module.i_typedef is not None:
            enum_statement = type_module.i_typedef.search_one('type')
            if enum_statement is not None and enum_statement.arg == 'enumeration':
                enum_val = self._get_enum_value_exec(enum_name, enum_statement)
        elif type_module.arg == 'enumeration':
            enum_val = self._get_enum_value_exec(enum_name, type_module)
        return enum_val

    def translate_enum_to_proto(self, proto_module, enum_name, proto_file_name, yang_module):
        type_module = yang_module.search_one('type')
        if yang_module.i_leafref is not None:
            return self.translate_enum_to_proto(proto_module, enum_name, proto_file_name,
                                                type_module.i_type_spec.i_target_node)
        if type_module.i_type_spec.name == 'bits':
            for enum_val in type_module.i_type_spec.bits:
                sub_proto_module = MessageObj(proto_file_name)
                sub_proto_module.scope = 'simple'
                sub_proto_module.name = self.get_enumval_type(str(enum_name), str(enum_val[0]), proto_module.proto_xpath)
                sub_proto_module.parent = proto_module
                sub_proto_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_proto_module.name)
                proto_module.statement.append(sub_proto_module)
        elif type_module.i_type_spec.name == 'enumeration':
            for enum_val in type_module.i_type_spec.enums:
                sub_proto_module = MessageObj(proto_file_name)
                sub_proto_module.scope = 'simple'
                sub_proto_module.name = self.get_enumval_type(str(enum_name), str(enum_val[0]), proto_module.proto_xpath)
                #sub_proto_module.enum_val = self.get_enum_value(str(enum_val[0]), type_module)
                sub_proto_module.enum_val = enum_val[1]
                if sub_proto_module.enum_val != -1:
                    sub_proto_module.num = str(sub_proto_module.enum_val)
                sub_proto_module.parent = proto_module
                sub_proto_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_proto_module.name)
                proto_module.statement.append(sub_proto_module)

        return proto_module

    def translate_decimal64_to_proto(self, proto_file_name, proto_file):
        proto_module = DefinionObj(proto_file_name)
        proto_module.keyword = 'message'
        name = 'Decimal64'
        proto_module.name = name
        proto_file.definions[name] = proto_module

        # 例：int64  value = 1;
        int64_sub_proto_module = MessageObj(proto_file_name)
        int64_sub_proto_module.scope = 'simple'
        int64_sub_proto_module.name = 'value'
        int64_sub_proto_module.type = 'int64'
        int64_sub_proto_module.num = '1'
        int64_sub_proto_module.parent = proto_module
        int64_sub_proto_module.yang_node_name = 'value'
        proto_module.statement.append(int64_sub_proto_module)
        # 例：uint32 fraction_digits = 2;
        uint32_sub_proto_module = MessageObj(proto_file_name)
        uint32_sub_proto_module.scope = 'simple'
        uint32_sub_proto_module.name = 'fraction_digits'
        uint32_sub_proto_module.type = 'uint32'
        uint32_sub_proto_module.num = '2'
        uint32_sub_proto_module.parent = proto_module
        uint32_sub_proto_module.yang_node_name = 'fraction-digits'
        proto_module.statement.append(uint32_sub_proto_module)

        return proto_module

    def get_out_union_type(self, type_module):
        if type_module.i_type_spec.name:
            pass

    def trans_yangleaf_to_proto(self, proto_module, yang_module):
        proto_module.type = self.get_type_name_for_leaf(yang_module)
        if self.bits_scope_change(yang_module):
            proto_module.scope = 'repeated'
        return proto_module

    def bits_scope_change(self, yang_module):
        type_module = yang_module.search_one('type')
        if yang_module.i_leafref is None:
            try:
                if type_module.i_type_spec.name == 'bits':
                    return True
            except:
                logging.error("import in yang file:%s related to the node type is missing." % yang_module.i_module.arg)
        return False

    def trans_node_name(self, arg, father_proto_xpath, is_leaf_node=False):
        # calculate if same node occured
        arg = arg.replace('-', '_')
        name = self.calculate_proto_name(father_proto_xpath, arg, is_leaf_node)

        return name

    def trans_module_type(self, arg):
        return self.first_letter_upper(arg)

    # 使用uses grouping对choice进行augment，并且augment的内容是简写的case(参考RFC7950 7.9.2 choice的case声明)
    # （即分支包含单个的”anydata”、“anyxml”、”choice”、“container”、“leaf”、“list”或者“leaf-list”声明，”case”声明可以忽略）
    # example：这里leaf c-aaa等同于case c-aaa{ leaf c-aaa }
    #      例：uses choice:choice-subnet {
    #        例：augment "test-choice/subnet" {
    #            例：leaf c-aaa{
    #                例：type string;
    #            }
    #         }
    #       }
    def is_augment_shorthand_case(self, yang_node):
        if yang_node.keyword == 'case' and len(yang_node.i_children) == 1:
            yang_node_pos = yang_node.pos
            child_node_pos = yang_node.i_children[0].pos
            if yang_node_pos.line == child_node_pos.line and yang_node_pos.ref == child_node_pos.ref:
                if yang_node.i_children[0].i_augment.keyword == 'augment':
                    logging.info('augment shorthand case, yang_node:%s' % yang_node.arg)
                    return True
        return False

    def find_yang_node_order_num(self, yang_node):
        children_list = yang_node.parent.i_children
        i = 0
        for children in children_list:
            if children.arg == yang_node.arg:
                return i
            if children.keyword == 'choice':
                i += len(children.search('case'))
                continue
            i += 1
        return 0

    def arrange_message_type(self, container_yang_module, proto_file_name, proto_module):
        sub_message_name = self.trans_node_name(container_yang_module.arg, proto_module.proto_xpath)
        sub_message_type = self.get_out_message_definion_name(container_yang_module.arg, proto_module.proto_xpath)
        sub_define_module = DefinionObj(proto_file_name)
        sub_define_module.keyword = 'message'
        sub_define_module.parent = proto_module

        sub_define_module.name = sub_message_type
        sub_define_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_type)
        yang_node_order_num = self.find_yang_node_order_num(container_yang_module)
        sub_define_module.yang_node_order_num = yang_node_order_num + proto_module.yang_node_order_num

        proto_module.definions[sub_message_type] = self._parse_statement(
            proto_file_name, sub_define_module, container_yang_module)

        sub_proto_module = MessageObj(proto_file_name)
        sub_proto_module.name = sub_message_name
        sub_proto_module.parent = proto_module
        sub_proto_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_type)
        sub_proto_module.yang_node_order_num = yang_node_order_num
        sub_proto_module.yang_node_name = container_yang_module.arg

        sub_proto_module.type = proto_module.definions[sub_message_type]
        if container_yang_module.keyword == 'case' and proto_module.keyword == 'oneof':
            sub_define_module.name = '%s_%s' % (proto_module.name, sub_message_type)
            sub_define_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_define_module.name)
            sub_proto_module.name = '%s_%s' %(container_yang_module.parent.arg.replace('-', '_'), sub_proto_module.name)
            sub_proto_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_proto_module.name )
            sub_proto_module.yang_node_order_num = yang_node_order_num + proto_module.yang_node_order_num
            proto_module.parent.statement.append(sub_proto_module)
            proto_module.parent.definions[sub_define_module.name] = proto_module.definions[sub_message_type]
            sub_proto_module.type = proto_module.parent.definions[sub_define_module.name]
            del proto_module.definions[sub_message_type]

        proto_module.statement.append(sub_proto_module)
        return proto_module

    def arrange_repeated_type(self, container_yang_module, proto_file_name, proto_module):
        sub_message_name = self.trans_node_name(container_yang_module.arg, proto_module.proto_xpath)
        sub_message_type = self.get_out_message_definion_name(container_yang_module.arg, proto_module.proto_xpath)
        sub_proto_module = MessageObj(proto_file_name)
        sub_proto_module.scope = 'repeated'
        sub_proto_module.name = sub_message_name
        sub_proto_module.parent = proto_module
        sub_proto_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_type)
        yang_node_order_num = self.find_yang_node_order_num(container_yang_module)
        sub_proto_module.yang_node_order_num = yang_node_order_num
        sub_proto_module.yang_node_name = container_yang_module.arg

        if container_yang_module.keyword == 'leaf-list':
            sub_proto_module.type = self.get_type_name_for_leaf(container_yang_module)
            if sub_proto_module.type == 'enum':
                sub_define_module = DefinionObj(proto_file_name)
                sub_define_module.keyword = 'enum'
                sub_define_module.parent = proto_module
                sub_define_module.name = sub_message_type
                sub_define_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_type)
                sub_define_module.yang_node_order_num = container_yang_module.pos.line
                proto_module.definions[sub_message_type] = self.translate_enum_to_proto(
                    sub_define_module, sub_message_type, proto_file_name, container_yang_module)
                sub_proto_module.type = proto_module.definions[sub_message_type]
        else:
            sub_define_module = DefinionObj(proto_file_name)
            sub_define_module.keyword = 'message'
            sub_define_module.parent = proto_module
            sub_define_module.name = sub_message_type
            sub_define_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_type)
            sub_define_module.yang_node_order_num = yang_node_order_num

            proto_module.definions[sub_message_type] = self._parse_statement(
                proto_file_name, sub_define_module, container_yang_module)
            sub_proto_module.type = proto_module.definions[sub_message_type]
        proto_module.statement.append(sub_proto_module)
        return proto_module

    def arrange_simple_type(self, leaf_yang_module, proto_file_name, proto_module):
        is_leaf_node = False
        if leaf_yang_module.keyword == 'leaf':
            is_leaf_node = True
        sub_message_name = self.trans_node_name(leaf_yang_module.arg, proto_module.proto_xpath, is_leaf_node)
        sub_proto_module = MessageObj(proto_file_name)
        sub_proto_module.scope = 'simple'
        sub_proto_module.name = sub_message_name
        sub_proto_module.parent = proto_module
        sub_proto_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_name)
        sub_message_type = self.get_out_message_definion_name(leaf_yang_module.arg, proto_module.proto_xpath)
        yang_node_order_num = self.find_yang_node_order_num(leaf_yang_module)
        sub_proto_module.yang_node_order_num = yang_node_order_num
        sub_proto_module.yang_node_name = leaf_yang_module.arg
        if is_leaf_node:
            sub_proto_module = self.trans_yangleaf_to_proto(sub_proto_module, leaf_yang_module)
            if sub_proto_module.type == 'enum':
                sub_define_module = DefinionObj(proto_file_name)
                sub_define_module.keyword = 'enum'
                sub_define_module.parent = proto_module
                sub_define_module.name = sub_message_type
                sub_define_module.proto_xpath = '%s/%s' % (proto_module.proto_xpath, sub_message_type)
                sub_define_module.yang_node_order_num = leaf_yang_module.pos.line
                proto_module.definions[sub_message_type] = self.translate_enum_to_proto(
                    sub_define_module, sub_message_type, proto_file_name, leaf_yang_module)
                sub_proto_module.type = proto_module.definions[sub_message_type]
        elif leaf_yang_module.keyword in yang_keyword_not_support:
            return proto_module
        else:
            logging.info('%s is unreachable' % leaf_yang_module.keyword)
            sub_proto_module.type = sub_message_type
        proto_module.statement.append(sub_proto_module)
        return proto_module

    def arrange_oneof_type(self, container_yang_module, proto_file_name, proto_module):
        if container_yang_module.keyword in oneof_type:
            sub_message_name = self.trans_node_name(container_yang_module.arg, proto_module.proto_xpath)
            sub_message_type = self.get_out_message_definion_name(container_yang_module.arg, proto_module.proto_xpath)
            logging.info('generating oneof message node:%s', container_yang_module.arg)
            sub_proto_module = MessageObj(proto_file_name)
            sub_proto_module.scope = ''
            sub_proto_module.name = sub_message_name
            sub_proto_module.parent = proto_module
            sub_proto_module.proto_xpath = '%s' % proto_module.proto_xpath
            yang_node_order_num = self.find_yang_node_order_num(container_yang_module)
            sub_proto_module.yang_node_order_num = yang_node_order_num
            sub_proto_module.yang_node_name = container_yang_module.arg

            logging.info('arrange container type:yang_node:%s, define_node:%s, order num:%d'
                         % (container_yang_module.arg, sub_proto_module.name, sub_proto_module.yang_node_order_num))

            sub_define_module = DefinionObj(proto_file_name)
            sub_define_module.keyword = 'oneof'
            sub_define_module.parent = proto_module
            sub_define_module.name = sub_message_type
            sub_define_module.proto_xpath = '%s' % proto_module.proto_xpath
            sub_define_module.yang_node_order_num = yang_node_order_num

            logging.info('arrange container type:yang_node:%s, define_node:%s, order num:%d'
                         % (container_yang_module.arg, sub_define_module.name, sub_define_module.yang_node_order_num))

            proto_module.definions[sub_message_type] = self._parse_statement(
                proto_file_name, sub_define_module, container_yang_module)
            sub_proto_module.type = proto_module.definions[sub_message_type]
            proto_module.statement.append(sub_proto_module)
        return proto_module

    @staticmethod
    def calculate_proto_name(father_proto_xpath, proto_name, is_leaf_node=False):
        node_name_list = father_proto_xpath.upper().split('/')
        # leaf node can not be ext keep word, other structure can not be
        # ext keep words and origin keep words
        if proto_name.upper() in node_name_list:
            return '%s%s' %(proto_name, str(len(node_name_list)))
        if proto_name.replace('_', '').upper() in ext_keep_word:
            return '%s%s' % (proto_name, str(len(node_name_list)))
        if not is_leaf_node and proto_name.replace('_', '').upper() in origin_keep_word:
            return '%s%s' % (proto_name, str(len(node_name_list)))
        return proto_name

    def _parse_statement(self, proto_file_name, proto_node, yang_node):
        if proto_node.parent is None:
            proto_node.is_top = True
        split_node = self.split_yang_node(yang_node)
        for container_yang_node in split_node['message']:
            proto_node = self.arrange_message_type(container_yang_node, proto_file_name, proto_node)

        for container_yang_node in split_node['oneof']:
            proto_node = self.arrange_oneof_type(container_yang_node, proto_file_name, proto_node)

        for container_yang_node in split_node['repeated']:
            proto_node = self.arrange_repeated_type(container_yang_node, proto_file_name, proto_node)

        for leaf_yang_node in split_node['simple']:
            proto_node = self.arrange_simple_type(leaf_yang_node, proto_file_name, proto_node)

        return proto_node

    def translate_heads(self, yang_name, module, proto_file):
        for annotation in annotations:
            annotation_obj = AnnotationObj(yang_name, annotation, proto_file.pos.line)
            proto_file.i_annotations.append(annotation_obj)
        annotation_module = '// module %s\n' % module.arg
        annotation_obj = AnnotationObj(yang_name, annotation_module, proto_file.pos.line)
        proto_file.i_annotations.append(annotation_obj)
        revision_modules = module.search('revision')
        for revision_module in revision_modules:
            annotation_module = '// revision %s\n' % revision_module.arg
            annotation_obj = AnnotationObj(yang_name, annotation_module, proto_file.pos.line)
            proto_file.i_annotations.append(annotation_obj)
        namespace_modules = module.search('namespace')
        for revision_module in namespace_modules:
            annotation_module = '// namespace %s\n' % revision_module.arg
            annotation_obj = AnnotationObj(yang_name, annotation_module, proto_file.pos.line)
            proto_file.i_annotations.append(annotation_obj)

        proto_file.i_package = module.arg.replace('-', '_')
        return

    def set_annotations(self, annotation_obj, str, pos):
        annotation_obj.str = str
        annotation_obj.ref = pos



    def generate_proto_custom(self, yang_module):
        if not yang_module.i_children:
            return
        if not yang_module.keyword == 'module':
            return
        proto_file_name = os.path.join(self.out_proto_directory, ''.join([yang_module.arg, '.proto']))

        logging.info('generateing proto file:%s', proto_file_name)
        proto_file = ProtoFile(proto_file_name, proto_version)
        self.translate_heads(proto_file_name, yang_module, proto_file)
        self.translate_services()
        self.translate_messages(proto_file_name, yang_module, proto_file)
        return proto_file

    def get_yang_file_name(self, yang_directory, module_name):
        yang_name = ''.join([module_name, '.yang'])
        if yang_name in os.listdir(yang_directory):
            return os.path.join(yang_directory, yang_name)
        return ''

    def get_proto_file(self, proto_file_name, ptx):
        text = ''
        try:
            with open(proto_file_name, "r", encoding="utf-8") as fd:
                text = fd.read()
        except Exception:
            sys.stderr.write("can not open the file %s" % proto_file_name)
            logging.error("can not open the file %s" % proto_file_name)
        module = ptx.add_modules(proto_file_name, text)
        for error in ptx.errors:
            logging.info('%s in %s' % (str(str(error[2])), str(error[0])))
        if ptx.errors:
            logging.info("proto file %s parsing break." % proto_file_name)
            return None
        return module

    def get_module_for_out(self, src_proto_file_name, ptx, module_name, yang_proto_file):
        if os.path.isfile(src_proto_file_name):
            src_proto_module = self.get_proto_file(src_proto_file_name, ptx)
            if src_proto_module:
                for definion in src_proto_module.definions.values():
                    del_same_node_in_proto(definion)
            ret_val = merge_proto_module(yang_proto_file, src_proto_module)
            if ret_val == 0:
                m = proto_writer.Proto_writer(src_proto_module)
                return m
        m = proto_writer.Proto_writer(yang_proto_file)
        return m



    def translate_process(self, yang_module, ptx):
        proto_file = self.generate_proto_custom(yang_module)
        if not proto_file:
            return
        m = self.get_module_for_out(proto_file.ref, ptx, yang_module.arg, proto_file)
        if m:
            m.output_proto_file()

    def copy_old_proto(self):
        if not self.src_proto_directory or not os.path.isdir(self.src_proto_directory):
            logging.info('proto directory is null')
            return

        for proto_file in os.listdir(self.src_proto_directory):
            if not proto_file.endswith('.proto'):
                continue
            old_proto = os.path.join(self.src_proto_directory, proto_file)
            if proto_file not in os.listdir(self.out_proto_directory):
                shutil.copy(old_proto, self.out_proto_directory)
        return
    def file_is_exist(self,file,dir=None):
        if dir is None:
            return os.path.exists(file)
        if not os.path.isdir(dir):
            if os.path.samefile(file,dir):
                return True
            else:
                return False
        for sub_dir in os.listdir(dir):
            if os.path.isfile(sub_dir):
                if os.path.samefile(file,sub_dir):
                    return True
            else:
                if self.file_is_exist(file,os.path.join(dir,sub_dir)):
                    return True
        return False
    def start_translate(self):
        if not os.path.exists(self.log_directory):
            os.makedirs(self.log_directory)

        log_file_name = os.path.join(self.log_directory, ''.join([rep_revs_log_file, '.log']))
        if os.path.exists(log_file_name):
            os.remove(log_file_name)

        logging.basicConfig(filename=log_file_name,
                            format='%(asctime)s,%(levelname)s,%(filename)s,%(lineno)d:%(message)s',
                            level=logging.DEBUG,

                            )
        logging.info(
            "==============================START YANG BASE LINE PROCESS=========================================")
        logging.info("option is: \n yang_path: %s\n"
                     " log_file_name: %s\n"
                     " out_proto_directory: %s\n"
                     " src_proto_directory: %s\n"
                     " yang_dependencies: %s\n",
                     self.yang_path, log_file_name,
                     self.out_proto_directory, self.src_proto_directory,self.yang_dependencies,
                     )
        sys.stderr.write("start translating yang to proto \n")
        ptx = hpproto.Context()

        import platform
        if platform.system() == 'Windows':
            split_tag = ';'
        else:
            split_tag = ':'
        if os.path.isdir(self.yang_path):
            dir_yang_str = ''.join([str(self.yang_path), split_tag, str(self.yang_dependencies)])
            ctx = yang_util.init_ctx(dir_yang_str)
        else:
            ctx = yang_util.init_ctx(self.yang_dependencies)
        self.copy_old_proto()
        #parse yang path
        yang_util.parse_yang_modules(self.yang_path,ctx)
        if os.path.exists(self.yang_dependencies):
            yang_util.parse_yang_modules(self.yang_dependencies, ctx)
        if yang_util.yang_module_has_error(ctx):
            yang_util.print_yang_errors(ctx)
            return
        for module_name,rev in ctx.modules:
            yang_module = ctx.get_module(module_name,rev)
            src_yang_file = yang_module.pos.ref
            if self.file_is_exist(src_yang_file,self.yang_path):
                self.translate_process(yang_module, ptx)

        # if self.module_name:
        #     if yang_util.yang_module_has_error(ctx,self.module_name):
        #         yang_util.print_yang_errors(ctx,self.module_name)
        #         return
        #     self.translate_process(self.module_name, ptx, ctx)
        # else:
        #     yang_modules = yang_util.get_out_main_module_yang_from_path(self.yang_directory)
        #     for yang_module in yang_modules:
        #         if yang_util.yang_module_has_error(ctx, yang_module):
        #             yang_util.print_yang_errors(ctx, yang_module)
        #             return
        #         self.translate_process(yang_module, ptx, ctx)
        #     self.copy_old_proto()
        return ptx

def parse_error_and_exit(filename):
    ret_val = 0
    try:
        with open(filename) as handle:
            error_context = handle.read()
        findre = re.compile(r",ERROR," , re.DOTALL)
        error_list = findre.findall(error_context)
        if len(error_list) == 0:
            ret_val = 0
        else:
            ret_val = 1
            print('YANG_TO_PROTO_ERROR-----')
    except Exception:
        return 1
    finally:
        return ret_val


if __name__ == "__main__":
    start = time.time()
    options = env_parse()
    p = Yang_proto_converter(options)
    ptx = p.start_translate()
    sys.stderr.write("end process !\n")
    end = time.time()
    sys.stderr.write("process time:" +str(end - start))
    #log_file_name = os.path.join(os.path.abspath(options.log_directory), ''.join([rep_revs_log_file, '.log']))
    sys.exit(0)
