Module pyvcli_replic

The pyvserv ihave/youhave replicator client.
Usage: pyvcli_replic.py [options]
  options:
    -d level  - Debug level 0-10
    -p        - Port to use (default: 6666)
    -P        - Second server port to use (default: 5555)
    -l login  - Login Name; Source server. default: 'admin'
    -s lpass  - Login Pass; default: '1234' (for !!testing only!!)
    -L login2 - Login Name; Target server. default: 'admin'
    -S lpass2 - Login Pass;  default: '1234'
    -b        - Begin time. Format: 'Y-m-d H:M' Default: now
    -i        - Interval in minutes. (Default: 1 day)
    -a        - Replicate all records. All server records.
    -v        - Verbose. More info on screen.
    -q        - Quiet. Less info on screen.
    -h        - Help. (this screen)
Expand source code
#!/usr/bin/env python

import sys
if sys.version_info[0] < 3:
    print("Python 2 is not supported as of 1/1/2020")
    sys.exit(1)

# ------------------------------------------------------------------------
# Test client for the pyserv project. Download file.

import os, sys, getopt, signal, select, socket, time, struct
import random, stat, datetime, atexit

# This repairs the path from local run to pip run.

try:
    from pyvcommon import support
    #print("sf", sf)
    # Get Parent of module root
    sf = os.path.dirname(support.__file__)
    sf = os.path.dirname(sf)
    sys.path.append(os.path.join(sf, "pyvcommon"))
    sys.path.append(os.path.join(sf, "pyvserver"))
except:
    #print("pathching")
    base = os.path.dirname(os.path.realpath(__file__))
    sys.path.append(os.path.join(base,  '..'))
    sys.path.append(os.path.join(base,  '..', "pyvcommon"))
    sys.path.append(os.path.join(base,  '..', "pyvserver"))
    from pyvcommon import support

import support, pycrypt, pyservsup, pyclisup
import pysyslog, comline, pyvhash

hand = None
hand2 = None
replic = 0

# ------------------------------------------------------------------------
# Globals

version = "1.0.0"

# ------------------------------------------------------------------------

phelpstr = '''\
The pyvserv ihave/youhave replicator client.
Usage: %s [options]
  options:
    -d level  - Debug level 0-10
    -p        - Port to use (default: 6666)
    -P        - Second server port to use (default: 5555)
    -l login  - Login Name; Source server. default: 'admin'
    -s lpass  - Login Pass; default: '1234' (for !!testing only!!)
    -L login2 - Login Name; Target server. default: 'admin'
    -S lpass2 - Login Pass;  default: '1234'
    -b        - Begin time. Format: 'Y-m-d H:M' Default: now
    -i        - Interval in minutes. (Default: 1 day)
    -a        - Replicate all records. All server records.
    -v        - Verbose. More info on screen.
    -q        - Quiet. Less info on screen.
    -h        - Help. (this screen)''' % (os.path.basename(__file__))

__doc__ = "<pre>" + phelpstr + "</pre>"

def phelp():
    ''' Deliver help '''
    print(phelpstr)
    sys.exit(0)

def pversion():
    ''' Print version '''
    print( os.path.basename(sys.argv[0]), "Version", version)
    sys.exit(0)

    # option, var_name, initial_val, function
optarr = \
    ["d:",  "pgdebug",  0,          None],      \
    ["p:",  "port",     6666,       None],      \
    ["P:",  "port2",    5555,       None],      \
    ["l:",  "login",    "admin",    None],      \
    ["s:",  "lpass",    "1234",     None],      \
    ["L:",  "login2",   "admin",    None],      \
    ["S:",  "lpass2",   "1234",     None],      \
    ["b:",  "start",    "",         None],      \
    ["i:",  "inter",    0,          None],      \
    ["a",   "all",      0,          None],      \
    ["v",   "verbose",  0,          None],      \
    ["q",   "quiet",    0,          None],      \
    ["V",   None,       None,       pversion],  \
    ["h",   None,       None,       phelp]      \

conf = comline.Config(optarr)
conf2 = comline.Config(optarr)

def scanall(hand, hand2):

    ''' Scan all data fron source server '''

    cresp = hand.client(["rsize", "vote",], conf.sess_key)
    #print ("Server  rsize response:", cresp)
    if cresp[0] != "OK":
        print("Cannot get records size")
        return

    for aa in range(cresp[1]):
        got = hand.client(["rabs", "vote", str(aa)], conf.sess_key)
        #print("got:", got[3])
        if got[0] != "OK":
            print("error:", got)
            continue

        dec = hand.pb.decode_data(got[3][0][1])[0]
        if conf.pgdebug > 3:
            print("dec:", dec)
        pay = hand.pb.decode_data(dec['payload'])[0]
        #print("pay:", pay['PayLoad'])
        if conf.pgdebug > 2:
            print("payload:", pay)
        if conf.pgdebug > 1:
            print("pay:", pay['PayLoad'])

        put = hand2.client(["rput", "vote", pay, ], conf2.sess_key)
        if put[0] != "OK":
            #print("put:", put)
            # Duplicate is normal
            if not "Duplicate" in put[1]:
                print("on put:", put)
        else:
            global replic
            replic += 1
            if not conf.quiet:
                print("replicated:", pay['header'])


def scanfordays(handx, handx2):

    ''' Progressively dig down the timeline '''

    global hand, hand2
    hand    = handx
    hand2   = handx2

    # Set beginning date range
    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)

    maxdays = 5; dayx = 0

    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)
    dd = dd - datetime.timedelta(maxdays)

    # Scan which days have records
    while True:
        dd_beg = dd + datetime.timedelta(dayx)
        dd_end = dd_beg + datetime.timedelta(1)

        cresp = hand.client(["rcount", "vote", dd_beg.timestamp(),
                                        dd_end.timestamp()], conf.sess_key)
        if cresp[1] > 0:
            print("from:", dd_beg, "to:", dd_end);
            print ("Server rcount response:", cresp)

            if cresp[1] >= 100:
                scanhours(maxdays - dayx)
            else:
                print("getting day:", dd_beg, "-", dd_end)

        #if cresp[1] > 100:
        #    #print("record with more", cresp[1])
        #    pass

        dayx += 1
        # We end one day late to include all timezone deviations
        if dayx >= maxdays + 1:
            break

# Scan which hours of days that have records
def scanhours(dayx):

    ''' Progressively dig down the timeline, per hour '''

    # Set beginning date range
    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)
    dd = dd - datetime.timedelta(dayx)

    maxhours = 24; hourx = 0

    while True:
        dd_beg = dd + datetime.timedelta(0, hourx * 3600)
        dd_end = dd_beg + datetime.timedelta(0, 1 * 3600)

        cresp = hand.client(["rcount", "vote", dd_beg.timestamp(),
                                        dd_end.timestamp()], conf.sess_key)
        if cresp[1] > 0:
            print(" from:", dd_beg, "to:", dd_end);
            print(" Server hour rcount response:", cresp)
            if cresp[1] >= 100:
                scanminutes(dayx, hourx)
            else:
                print(" getting hour:", dd_beg, " - ", dd_end)
                cresp = hand.client(["rlist", "vote", dd_beg.timestamp(),
                                    dd_end.timestamp()], conf.sess_key)
        hourx += 1
        if hourx > maxhours:
            break

def scanminutes(dayx, hourx):

    ''' Progressively dig down the timeline per minute '''

    # Set beginning date range
    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)
    dd = dd - datetime.timedelta(dayx)
    dd = dd + datetime.timedelta(0, hourx * 3600)

    maxmins = 60; minx = 0

    while True:
        dd_beg = dd + datetime.timedelta(0, minx * 60)
        dd_end = dd_beg + datetime.timedelta(0, 60)

        cresp = hand.client(["rcount", "vote", dd_beg.timestamp(),
                                        dd_end.timestamp()], conf.sess_key)
        if cresp[1] > 0:
            print("    from:", dd_beg, "to:", dd_end);
            print("    Server min  rcount response:", cresp)

            if cresp[1] >= 100:
                print("    ----------- big rec count")
            else:
                #print("    getting minutes:", dd_beg, "-", dd_end)
                cresp = hand.client(["rlist", "vote", dd_beg.timestamp(),
                                    dd_end.timestamp()], conf.sess_key)
                if cresp[0] == "OK":
                    crespr = hand.client(["rget", "vote", cresp[1]], conf.sess_key)
                    #print("rget", crespr)
                    for aa in crespr[1]:
                        #print("aa", aa)
                        dec = hand.pb.decode_data(aa[1])[0]
                        #print("dec", dec)
                        print(dec['header'], dec['payload'])

                    #for aa in cresp[1]:
                    #    #print("head", aa)
                    #    #crespg = hand2.client(["rhave", "vote", aa], conf2.sess_key)
                    #    #print("rhave", crespg)
                    #    if 1: #crespg[0] != "OK":
                    #        crespr = hand.client(["rget", "vote", aa], conf.sess_key)
                    #        #print("Rec", crespr[1])
                    #        cresp3 = hand2.client(
                    #                ["rput", "vote", crespr[1][0], ], conf2.sess_key)
                    #        if cresp3[0] == 'OK':
                    #            print("rput", cresp3)
                #print(cresp)

        minx += 1
        if minx > maxmins:
            break

# ------------------------------------------------------------------------

def    mainfunct():

    ''' Called by the SETUP script '''

    args = conf.comline(sys.argv[1:])

    pyclisup.verbose = conf.verbose
    pyclisup.pgdebug = conf.pgdebug

    if len(args) == 0:
        ip = '127.0.0.1'
        ip2 = '127.0.0.1'
    elif len(args) == 1:
        print("Must specify second server for replication.")
        sys.exit()
    else:
        ip = args[0]
        ip2 = args[1]

    hand = pyclisup.CliSup()
    hand.verbose = conf.verbose
    hand.pgdebug = conf.pgdebug

    try:
        respc = hand.connect(ip, conf.port)
    except:
        print( "Cannot connect to:", ip + ":" + str(conf.port), sys.exc_info()[1])
        sys.exit(1)

    hand2 = pyclisup.CliSup()
    hand2.verbose = conf.verbose
    hand2.pgdebug = conf.pgdebug

    atexit.register(pyclisup.atexit_func, hand, conf)

    ret = hand.start_session(conf)
    if ret[0] != "OK":
        print("Error on setting session:", resp3[1])
        hand.client(["quit"])
        hand.close();
        sys.exit(0)

    #resp3 = hand.client(["hello", ],  conf.sess_key, False)
    #print("Hello  Resp:", resp3)

    try:
        respc = hand2.connect(ip, conf.port2)
    except:
        print( "Cannot connect to second server:", ip2 + ":" + str(conf.port2), sys.exc_info()[1])
        sys.exit(1)

    atexit.register(pyclisup.atexit_func, hand2, conf2)

    ret2 = hand2.start_session(conf2)
    if ret2[0] != "OK":
        print("Error on setting session2:", ret2[1])
        hand2.client(["quit"])
        hand2.close();
        sys.exit(0)

    #resp3 = hand2.client(["hello", ],  conf2.sess_key, False)
    #print("Hello2 Resp:", resp3)

    cresp = hand.login(conf.login, conf.lpass, conf)
    #print ("Server login response:", cresp)
    if cresp[0] != "OK":
        print("Cannot login to server", cresp)
        sys.exit(0)

    if conf.verbose:
        print ("Server  login response:", cresp)

    cresp = hand.client(["rsize", "vote",], conf.sess_key)
    if not conf.quiet:
        print ("Server  rsize response:", cresp)

    cresp = hand2.login(conf.login2, conf.lpass2, conf2)
    if cresp[0] != "OK":
        print("Cannot login to server2", cresp)
        sys.exit(0)

    if conf.verbose:
        print ("Server2 login response:", cresp)

    cresp2 = hand2.client(["rsize", "vote",], conf2.sess_key)
    if not conf.quiet:
        print ("Server2 rsize response:", cresp2)

    if conf.all:
        scanall(hand, hand2)
    else:
        scanfordays(hand, hand2)

    print("Replicated", replic, "records.")

if __name__ == '__main__':
    mainfunct()

# EOF

Functions

def mainfunct()

Called by the SETUP script

Expand source code
def    mainfunct():

    ''' Called by the SETUP script '''

    args = conf.comline(sys.argv[1:])

    pyclisup.verbose = conf.verbose
    pyclisup.pgdebug = conf.pgdebug

    if len(args) == 0:
        ip = '127.0.0.1'
        ip2 = '127.0.0.1'
    elif len(args) == 1:
        print("Must specify second server for replication.")
        sys.exit()
    else:
        ip = args[0]
        ip2 = args[1]

    hand = pyclisup.CliSup()
    hand.verbose = conf.verbose
    hand.pgdebug = conf.pgdebug

    try:
        respc = hand.connect(ip, conf.port)
    except:
        print( "Cannot connect to:", ip + ":" + str(conf.port), sys.exc_info()[1])
        sys.exit(1)

    hand2 = pyclisup.CliSup()
    hand2.verbose = conf.verbose
    hand2.pgdebug = conf.pgdebug

    atexit.register(pyclisup.atexit_func, hand, conf)

    ret = hand.start_session(conf)
    if ret[0] != "OK":
        print("Error on setting session:", resp3[1])
        hand.client(["quit"])
        hand.close();
        sys.exit(0)

    #resp3 = hand.client(["hello", ],  conf.sess_key, False)
    #print("Hello  Resp:", resp3)

    try:
        respc = hand2.connect(ip, conf.port2)
    except:
        print( "Cannot connect to second server:", ip2 + ":" + str(conf.port2), sys.exc_info()[1])
        sys.exit(1)

    atexit.register(pyclisup.atexit_func, hand2, conf2)

    ret2 = hand2.start_session(conf2)
    if ret2[0] != "OK":
        print("Error on setting session2:", ret2[1])
        hand2.client(["quit"])
        hand2.close();
        sys.exit(0)

    #resp3 = hand2.client(["hello", ],  conf2.sess_key, False)
    #print("Hello2 Resp:", resp3)

    cresp = hand.login(conf.login, conf.lpass, conf)
    #print ("Server login response:", cresp)
    if cresp[0] != "OK":
        print("Cannot login to server", cresp)
        sys.exit(0)

    if conf.verbose:
        print ("Server  login response:", cresp)

    cresp = hand.client(["rsize", "vote",], conf.sess_key)
    if not conf.quiet:
        print ("Server  rsize response:", cresp)

    cresp = hand2.login(conf.login2, conf.lpass2, conf2)
    if cresp[0] != "OK":
        print("Cannot login to server2", cresp)
        sys.exit(0)

    if conf.verbose:
        print ("Server2 login response:", cresp)

    cresp2 = hand2.client(["rsize", "vote",], conf2.sess_key)
    if not conf.quiet:
        print ("Server2 rsize response:", cresp2)

    if conf.all:
        scanall(hand, hand2)
    else:
        scanfordays(hand, hand2)

    print("Replicated", replic, "records.")
def phelp()

Deliver help

Expand source code
def phelp():
    ''' Deliver help '''
    print(phelpstr)
    sys.exit(0)
def pversion()

Print version

Expand source code
def pversion():
    ''' Print version '''
    print( os.path.basename(sys.argv[0]), "Version", version)
    sys.exit(0)

    # option, var_name, initial_val, function
def scanall(hand, hand2)

Scan all data fron source server

Expand source code
def scanall(hand, hand2):

    ''' Scan all data fron source server '''

    cresp = hand.client(["rsize", "vote",], conf.sess_key)
    #print ("Server  rsize response:", cresp)
    if cresp[0] != "OK":
        print("Cannot get records size")
        return

    for aa in range(cresp[1]):
        got = hand.client(["rabs", "vote", str(aa)], conf.sess_key)
        #print("got:", got[3])
        if got[0] != "OK":
            print("error:", got)
            continue

        dec = hand.pb.decode_data(got[3][0][1])[0]
        if conf.pgdebug > 3:
            print("dec:", dec)
        pay = hand.pb.decode_data(dec['payload'])[0]
        #print("pay:", pay['PayLoad'])
        if conf.pgdebug > 2:
            print("payload:", pay)
        if conf.pgdebug > 1:
            print("pay:", pay['PayLoad'])

        put = hand2.client(["rput", "vote", pay, ], conf2.sess_key)
        if put[0] != "OK":
            #print("put:", put)
            # Duplicate is normal
            if not "Duplicate" in put[1]:
                print("on put:", put)
        else:
            global replic
            replic += 1
            if not conf.quiet:
                print("replicated:", pay['header'])
def scanfordays(handx, handx2)

Progressively dig down the timeline

Expand source code
def scanfordays(handx, handx2):

    ''' Progressively dig down the timeline '''

    global hand, hand2
    hand    = handx
    hand2   = handx2

    # Set beginning date range
    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)

    maxdays = 5; dayx = 0

    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)
    dd = dd - datetime.timedelta(maxdays)

    # Scan which days have records
    while True:
        dd_beg = dd + datetime.timedelta(dayx)
        dd_end = dd_beg + datetime.timedelta(1)

        cresp = hand.client(["rcount", "vote", dd_beg.timestamp(),
                                        dd_end.timestamp()], conf.sess_key)
        if cresp[1] > 0:
            print("from:", dd_beg, "to:", dd_end);
            print ("Server rcount response:", cresp)

            if cresp[1] >= 100:
                scanhours(maxdays - dayx)
            else:
                print("getting day:", dd_beg, "-", dd_end)

        #if cresp[1] > 100:
        #    #print("record with more", cresp[1])
        #    pass

        dayx += 1
        # We end one day late to include all timezone deviations
        if dayx >= maxdays + 1:
            break
def scanhours(dayx)

Progressively dig down the timeline, per hour

Expand source code
def scanhours(dayx):

    ''' Progressively dig down the timeline, per hour '''

    # Set beginning date range
    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)
    dd = dd - datetime.timedelta(dayx)

    maxhours = 24; hourx = 0

    while True:
        dd_beg = dd + datetime.timedelta(0, hourx * 3600)
        dd_end = dd_beg + datetime.timedelta(0, 1 * 3600)

        cresp = hand.client(["rcount", "vote", dd_beg.timestamp(),
                                        dd_end.timestamp()], conf.sess_key)
        if cresp[1] > 0:
            print(" from:", dd_beg, "to:", dd_end);
            print(" Server hour rcount response:", cresp)
            if cresp[1] >= 100:
                scanminutes(dayx, hourx)
            else:
                print(" getting hour:", dd_beg, " - ", dd_end)
                cresp = hand.client(["rlist", "vote", dd_beg.timestamp(),
                                    dd_end.timestamp()], conf.sess_key)
        hourx += 1
        if hourx > maxhours:
            break
def scanminutes(dayx, hourx)

Progressively dig down the timeline per minute

Expand source code
def scanminutes(dayx, hourx):

    ''' Progressively dig down the timeline per minute '''

    # Set beginning date range
    dd = datetime.datetime.now()
    dd = dd.replace(hour=0, minute=0, second=0, microsecond=0)
    dd = dd - datetime.timedelta(dayx)
    dd = dd + datetime.timedelta(0, hourx * 3600)

    maxmins = 60; minx = 0

    while True:
        dd_beg = dd + datetime.timedelta(0, minx * 60)
        dd_end = dd_beg + datetime.timedelta(0, 60)

        cresp = hand.client(["rcount", "vote", dd_beg.timestamp(),
                                        dd_end.timestamp()], conf.sess_key)
        if cresp[1] > 0:
            print("    from:", dd_beg, "to:", dd_end);
            print("    Server min  rcount response:", cresp)

            if cresp[1] >= 100:
                print("    ----------- big rec count")
            else:
                #print("    getting minutes:", dd_beg, "-", dd_end)
                cresp = hand.client(["rlist", "vote", dd_beg.timestamp(),
                                    dd_end.timestamp()], conf.sess_key)
                if cresp[0] == "OK":
                    crespr = hand.client(["rget", "vote", cresp[1]], conf.sess_key)
                    #print("rget", crespr)
                    for aa in crespr[1]:
                        #print("aa", aa)
                        dec = hand.pb.decode_data(aa[1])[0]
                        #print("dec", dec)
                        print(dec['header'], dec['payload'])

                    #for aa in cresp[1]:
                    #    #print("head", aa)
                    #    #crespg = hand2.client(["rhave", "vote", aa], conf2.sess_key)
                    #    #print("rhave", crespg)
                    #    if 1: #crespg[0] != "OK":
                    #        crespr = hand.client(["rget", "vote", aa], conf.sess_key)
                    #        #print("Rec", crespr[1])
                    #        cresp3 = hand2.client(
                    #                ["rput", "vote", crespr[1][0], ], conf2.sess_key)
                    #        if cresp3[0] == 'OK':
                    #            print("rput", cresp3)
                #print(cresp)

        minx += 1
        if minx > maxmins:
            break