Source code for cobbler.actions.reposync

"""
Builds out and synchronizes yum repo mirrors.
Initial support for rsync, perhaps reposync coming later.
"""

from builtins import range
from builtins import object
import os
import os.path
import pipes
import stat

HAS_YUM = True
try:
    import yum
except:
    HAS_YUM = False

from cobbler import clogger
from cobbler import utils
from cobbler import download_manager
from cobbler.utils import os_release


[docs]def repo_walker(top, func, arg): """ Directory tree walk with callback function. For each directory in the directory tree rooted at top (including top itself, but excluding '.' and '..'), call func(arg, dirname, fnames). dirname is the name of the directory, and fnames a list of the names of the files and subdirectories in dirname (excluding '.' and '..'). func may modify the fnames list in-place (e.g. via del or slice assignment), and walk will only recurse into the subdirectories whose names remain in fnames; this can be used to implement a filter, or to impose a specific order of visiting. No semantics are defined for, or required of, arg, beyond that arg is always passed to func. It can be used, e.g., to pass a filename pattern, or a mutable object designed to accumulate statistics. Passing None for arg is common. """ try: names = os.listdir(top) except os.error: return func(arg, top, names) for name in names: name = os.path.join(top, name) try: st = os.lstat(name) except os.error: continue if stat.S_ISDIR(st.st_mode): repo_walker(name, func, arg)
[docs]class RepoSync(object): """ Handles conversion of internal state to the tftpboot tree layout """ # ================================================================================== def __init__(self, collection_mgr, tries=1, nofail=False, logger=None): """ Constructor """ self.verbose = True self.api = collection_mgr.api self.collection_mgr = collection_mgr self.distros = collection_mgr.distros() self.profiles = collection_mgr.profiles() self.systems = collection_mgr.systems() self.settings = collection_mgr.settings() self.repos = collection_mgr.repos() self.rflags = self.settings.reposync_flags self.tries = tries self.nofail = nofail self.logger = logger self.dlmgr = download_manager.DownloadManager(self.collection_mgr, self.logger) if logger is None: self.logger = clogger.Logger() self.logger.info("hello, reposync") # ===================================================================
[docs] def run(self, name=None, verbose=True): """ Syncs the current repo configuration file with the filesystem. """ self.logger.info("run, reposync, run!") try: self.tries = int(self.tries) except: utils.die(self.logger, "retry value must be an integer") self.verbose = verbose report_failure = False for repo in self.repos: if name is not None and repo.name != name: # invoked to sync only a specific repo, this is not the one continue elif name is None and not repo.keep_updated: # invoked to run against all repos, but this one is off self.logger.info("%s is set to not be updated" % repo.name) continue repo_mirror = os.path.join(self.settings.webdir, "repo_mirror") repo_path = os.path.join(repo_mirror, repo.name) if not os.path.isdir(repo_path) and not repo.mirror.lower().startswith("rhn://"): os.makedirs(repo_path) # set the environment keys specified for this repo # save the old ones if they modify an existing variable env = repo.environment old_env = {} for k in list(env.keys()): self.logger.debug("setting repo environment: %s=%s" % (k, env[k])) if env[k] is not None: if os.getenv(k): old_env[k] = os.getenv(k) else: os.environ[k] = env[k] # which may actually NOT reposync if the repo is set to not mirror locally # but that's a technicality for x in range(self.tries + 1, 1, -1): success = False try: self.sync(repo) success = True break except: utils.log_exc(self.logger) self.logger.warning("reposync failed, tries left: %s" % (x - 2)) # cleanup/restore any environment variables that were # added or changed above for k in list(env.keys()): if env[k] is not None: if k in old_env: self.logger.debug("resetting repo environment: %s=%s" % (k, old_env[k])) os.environ[k] = old_env[k] else: self.logger.debug("removing repo environment: %s=%s" % (k, env[k])) del os.environ[k] if not success: report_failure = True if not self.nofail: utils.die(self.logger, "reposync failed, retry limit reached, aborting") else: self.logger.error("reposync failed, retry limit reached, skipping") self.update_permissions(repo_path) if report_failure: utils.die(self.logger, "overall reposync failed, at least one repo failed to synchronize")
# ==================================================================================
[docs] def sync(self, repo): """ Conditionally sync a repo, based on type. """ if repo.breed == "rhn": self.rhn_sync(repo) elif repo.breed == "yum": self.yum_sync(repo) elif repo.breed == "apt": self.apt_sync(repo) elif repo.breed == "rsync": self.rsync_sync(repo) elif repo.breed == "wget": self.wget_sync(repo) else: utils.die(self.logger, "unable to sync repo (%s), unknown or unsupported repo type (%s)" % (repo.name, repo.breed))
# ====================================================================================
[docs] def createrepo_walker(self, repo, dirname, fnames): """ Used to run createrepo on a copied Yum mirror. """ if os.path.exists(dirname) or repo['breed'] == 'rsync': utils.remove_yum_olddata(dirname) # add any repo metadata we can use mdoptions = [] if os.path.isfile("%s/.origin/repomd.xml" % (dirname)): if not HAS_YUM: utils.die(self.logger, "yum is required to use this feature") rmd = yum.repoMDObject.RepoMD('', "%s/.origin/repomd.xml" % (dirname)) if "group" in rmd.repoData: groupmdfile = rmd.getData("group").location[1] mdoptions.append("-g %s" % groupmdfile) if "prestodelta" in rmd.repoData: # need createrepo >= 0.9.7 to add deltas if utils.get_family() in ("redhat", "suse"): cmd = "/usr/bin/rpmquery --queryformat=%{VERSION} createrepo" createrepo_ver = utils.subprocess_get(self.logger, cmd) if utils.compare_versions_gt(createrepo_ver, "0.9.7"): mdoptions.append("--deltas") else: self.logger.error("this repo has presto metadata; you must upgrade createrepo to >= 0.9.7 first and then need to resync the repo through cobbler.") blended = utils.blender(self.api, False, repo) flags = blended.get("createrepo_flags", "(ERROR: FLAGS)") try: # BOOKMARK cmd = "createrepo %s %s %s" % (" ".join(mdoptions), flags, pipes.quote(dirname)) utils.subprocess_call(self.logger, cmd) except: utils.log_exc(self.logger) self.logger.error("createrepo failed.") del fnames[:] # we're in the right place
# ====================================================================================
[docs] def wget_sync(self, repo): """ Handle mirroring of directories using wget """ repo_mirror = repo.mirror.strip() if repo.rpm_list != "" and repo.rpm_list != []: self.logger.warning("--rpm-list is not supported for wget'd repositories") # FIXME: don't hardcode dest_path = os.path.join(self.settings.webdir + "/repo_mirror", repo.name) # FIXME: wrapper for subprocess that logs to logger cmd = "wget -N -np -r -l inf -nd -P %s %s" % (pipes.quote(dest_path), pipes.quote(repo_mirror)) rc = utils.subprocess_call(self.logger, cmd) if rc != 0: utils.die(self.logger, "cobbler reposync failed") repo_walker(dest_path, self.createrepo_walker, repo) self.create_local_file(dest_path, repo)
# ====================================================================================
[docs] def rsync_sync(self, repo): """ Handle copying of rsync:// and rsync-over-ssh repos. """ if not repo.mirror_locally: utils.die(self.logger, "rsync:// urls must be mirrored locally, yum cannot access them directly") if repo.rpm_list != "" and repo.rpm_list != []: self.logger.warning("--rpm-list is not supported for rsync'd repositories") # FIXME: don't hardcode dest_path = os.path.join(self.settings.webdir + "/repo_mirror", repo.name) spacer = "" if not repo.mirror.startswith("rsync://") and not repo.mirror.startswith("/"): spacer = "-e ssh" if not repo.mirror.strip().endswith("/"): repo.mirror = "%s/" % repo.mirror # FIXME: wrapper for subprocess that logs to logger cmd = "rsync -rltDv --copy-unsafe-links --delete-after %s --delete --exclude-from=/etc/cobbler/rsync.exclude %s %s" % (spacer, pipes.quote(repo.mirror), pipes.quote(dest_path)) rc = utils.subprocess_call(self.logger, cmd) if rc != 0: utils.die(self.logger, "cobbler reposync failed") repo_walker(dest_path, self.createrepo_walker, repo) self.create_local_file(dest_path, repo)
# ====================================================================================
[docs] def reposync_cmd(self): """ Determine reposync command """ cmd = None # reposync command if os.path.exists("/usr/bin/dnf"): cmd = "/usr/bin/dnf reposync" elif os.path.exists("/usr/bin/reposync"): cmd = "/usr/bin/reposync" else: # warn about not having yum-utils. We don't want to require it in the package because # Fedora 22+ has moved to dnf. utils.die(self.logger, "no /usr/bin/reposync found, please install yum-utils") return cmd
# ====================================================================================
[docs] def rhn_sync(self, repo): """ Handle mirroring of RHN repos. """ cmd = self.reposync_cmd() # reposync command has_rpm_list = False # flag indicating not to pull the whole repo # detect cases that require special handling if repo.rpm_list != "" and repo.rpm_list != []: has_rpm_list = True # create yum config file for use by reposync # FIXME: don't hardcode dest_path = os.path.join(self.settings.webdir + "/repo_mirror", repo.name) temp_path = os.path.join(dest_path, ".origin") if not os.path.isdir(temp_path): # FIXME: there's a chance this might break the RHN D/L case os.makedirs(temp_path) # how we invoke reposync depends on whether this is RHN content or not. # this is the somewhat more-complex RHN case. # NOTE: this requires that you have entitlements for the server and you give the mirror as rhn://$channelname if not repo.mirror_locally: utils.die(self.logger, "rhn:// repos do not work with --mirror-locally=1") if has_rpm_list: self.logger.warning("warning: --rpm-list is not supported for RHN content") rest = repo.mirror[6:] # everything after rhn:// cmd = "%s %s --repo=%s --download_path=%s" % (cmd, self.rflags, pipes.quote(rest), pipes.quote(self.settings.webdir + "/repo_mirror")) if repo.name != rest: args = {"name": repo.name, "rest": rest} utils.die(self.logger, "ERROR: repository %(name)s needs to be renamed %(rest)s as the name of the cobbler repository must match the name of the RHN channel" % args) if repo.arch == "i386": # counter-intuitive, but we want the newish kernels too repo.arch = "i686" if repo.arch != "": cmd = "%s -a %s" % (cmd, repo.arch) # now regardless of whether we're doing yumdownloader or reposync # or whether the repo was http://, ftp://, or rhn://, execute all queued # commands here. Any failure at any point stops the operation. if repo.mirror_locally: utils.subprocess_call(self.logger, cmd) # some more special case handling for RHN. # create the config file now, because the directory didn't exist earlier self.create_local_file(temp_path, repo, output=False) # now run createrepo to rebuild the index if repo.mirror_locally: repo_walker(dest_path, self.createrepo_walker, repo) # create the config file the hosts will use to access the repository. self.create_local_file(dest_path, repo)
# ==================================================================================== # This function translates yum repository options into the appropriate # options for python-requests
[docs] def gen_urlgrab_ssl_opts(self, yumopts): # use SSL options if specified in yum opts if 'sslclientkey' and 'sslclientcert' in yumopts: cert = (yumopts['sslclientcert'], yumopts['sslclientkey']) # note that the default of requests is to verify the peer and host # but the default here is NOT to verify them unless sslverify is # explicitly set to 1 in yumopts if 'sslverify' in yumopts: if yumopts['sslverify'] == 1: verify = True else: verify = False return (cert, verify)
# ====================================================================================
[docs] def yum_sync(self, repo): """ Handle copying of http:// and ftp:// yum repos. """ # create the config file the hosts will use to access the repository. repo_mirror = repo.mirror.strip() dest_path = os.path.join(self.settings.webdir + "/repo_mirror", repo.name.strip()) self.create_local_file(dest_path, repo) if not repo.mirror_locally: return cmd = self.reposync_cmd() # command to run has_rpm_list = False # flag indicating not to pull the whole repo # detect cases that require special handling if repo.rpm_list != "" and repo.rpm_list != []: has_rpm_list = True # create yum config file for use by reposync temp_path = os.path.join(dest_path, ".origin") if not os.path.isdir(temp_path): # FIXME: there's a chance this might break the RHN D/L case os.makedirs(temp_path) temp_file = self.create_local_file(temp_path, repo, output=False) if not has_rpm_list: # if we have not requested only certain RPMs, use reposync cmd = "%s %s --config=%s --repoid=%s --download_path=%s" % (cmd, self.rflags, temp_file, pipes.quote(repo.name), pipes.quote(self.settings.webdir + "/repo_mirror")) if repo.arch != "": if repo.arch == "x86": repo.arch = "i386" # FIX potential arch errors if repo.arch == "i386": # counter-intuitive, but we want the newish kernels too cmd = "%s -a i686" % (cmd) else: cmd = "%s -a %s" % (cmd, repo.arch) else: # create the output directory if it doesn't exist if not os.path.exists(dest_path): os.makedirs(dest_path) use_source = "" if repo.arch == "src": use_source = "--source" # older yumdownloader sometimes explodes on --resolvedeps # if this happens to you, upgrade yum & yum-utils extra_flags = self.settings.yumdownloader_flags cmd = "" if os.path.exists("/usr/bin/dnf"): cmd = "/usr/bin/dnf download" else: cmd = "/usr/bin/yumdownloader" cmd = "%s %s %s --disablerepo=* --enablerepo=%s -c %s --destdir=%s %s" % (cmd, extra_flags, use_source, pipes.quote(repo.name), temp_file, pipes.quote(dest_path), " ".join(repo.rpm_list)) # now regardless of whether we're doing yumdownloader or reposync # or whether the repo was http://, ftp://, or rhn://, execute all queued # commands here. Any failure at any point stops the operation. rc = utils.subprocess_call(self.logger, cmd) if rc != 0: utils.die(self.logger, "cobbler reposync failed") repodata_path = os.path.join(dest_path, "repodata") # grab repomd.xml and use it to download any metadata we can use proxies = None if repo.proxy == '<<inherit>>': proxies = self.settings.proxy_url_ext elif repo.proxy != '<<None>>' and repo.proxy != '': proxies = {'http': repo.proxy, 'https': repo.proxy} src = repo_mirror + "/repodata/repomd.xml" dst = temp_path + "/repomd.xml" (cert, verify) = self.gen_urlgrab_ssl_opts(repo.yumopts) try: self.dlmgr.download_file(src, dst, proxies, cert, verify) except Exception as e: utils.die(self.logger, "failed to fetch " + src + " " + e.args) # create our repodata directory now, as any extra metadata we're # about to download probably lives there if not os.path.isdir(repodata_path): os.makedirs(repodata_path) rmd = yum.repoMDObject.RepoMD('', "%s/repomd.xml" % (temp_path)) for mdtype in list(rmd.repoData.keys()): # don't download metadata files that are created by default if mdtype not in ["primary", "primary_db", "filelists", "filelists_db", "other", "other_db"]: mdfile = rmd.getData(mdtype).location[1] src = repo_mirror + "/" + mdfile dst = dest_path + "/" + mdfile try: self.dlmgr.download_file(src, dst, proxies, cert, verify) except Exception as e: utils.die(self.logger, "failed to fetch " + src + " " + e.args) # now run createrepo to rebuild the index if repo.mirror_locally: repo_walker(dest_path, self.createrepo_walker, repo)
# ====================================================================================
[docs] def apt_sync(self, repo): """ Handle copying of http:// and ftp:// debian repos. """ # warn about not having mirror program. mirror_program = "/usr/bin/debmirror" if not os.path.exists(mirror_program): utils.die(self.logger, "no %s found, please install it" % (mirror_program)) cmd = "" # command to run # detect cases that require special handling if repo.rpm_list != "" and repo.rpm_list != []: utils.die(self.logger, "has_rpm_list not yet supported on apt repos") if not repo.arch: utils.die(self.logger, "Architecture is required for apt repositories") # built destination path for the repo dest_path = os.path.join("/var/www/cobbler/repo_mirror", repo.name) if repo.mirror_locally: # NOTE: Dropping @@suite@@ replace as it is also dropped from # from manage_import_debian_ubuntu.py due that repo has no os_version # attribute. If it is added again it will break the Web UI! # mirror = repo.mirror.replace("@@suite@@",repo.os_version) mirror = repo.mirror idx = mirror.find("://") method = mirror[:idx] mirror = mirror[idx + 3:] idx = mirror.find("/") host = mirror[:idx] mirror = mirror[idx:] dists = ",".join(repo.apt_dists) components = ",".join(repo.apt_components) mirror_data = "--method=%s --host=%s --root=%s --dist=%s --section=%s" % (pipes.quote(method), pipes.quote(host), pipes.quote(mirror), pipes.quote(dists), pipes.quote(components)) rflags = "--nocleanup" for x in repo.yumopts: if repo.yumopts[x]: rflags += " %s %s" % (x, repo.yumopts[x]) else: rflags += " %s" % x cmd = "%s %s %s %s" % (mirror_program, rflags, mirror_data, dest_path) cmd = "%s %s %s %s" % (mirror_program, rflags, mirror_data, pipes.quote(dest_path)) if repo.arch == "src": cmd = "%s --source" % cmd else: arch = repo.arch if arch == "x86": arch = "i386" # FIX potential arch errors if arch == "x86_64": arch = "amd64" # FIX potential arch errors cmd = "%s --nosource -a %s" % (cmd, arch) # Set's an environment variable for subprocess, otherwise debmirror will fail # as it needs this variable to exist. # FIXME: might this break anything? So far it doesn't os.putenv("HOME", "/var/lib/cobbler") rc = utils.subprocess_call(self.logger, cmd) if rc != 0: utils.die(self.logger, "cobbler reposync failed")
[docs] def create_local_file(self, dest_path, repo, output=True): """ Creates Yum config files for use by reposync Two uses: (A) output=True, Create local files that can be used with yum on provisioned clients to make use of this mirror. (B) output=False, Create a temporary file for yum to feed into yum for mirroring """ # the output case will generate repo configuration files which are usable # for the installed systems. They need to be made compatible with --server-override # which means they are actually templates, which need to be rendered by a cobbler-sync # on per profile/system basis. if output: fname = os.path.join(dest_path, "config.repo") else: fname = os.path.join(dest_path, "%s.repo" % repo.name) self.logger.debug("creating: %s" % fname) if not os.path.exists(dest_path): utils.mkdir(dest_path) config_file = open(fname, "w+") if not output: config_file.write("[main]\nreposdir=/dev/null\n") config_file.write("[%s]\n" % repo.name) config_file.write("name=%s\n" % repo.name) optenabled = False optgpgcheck = False if output: if repo.mirror_locally: line = "baseurl=http://${http_server}/cobbler/repo_mirror/%s\n" % (repo.name) else: mstr = repo.mirror if mstr.startswith("/"): mstr = "file://%s" % mstr line = "baseurl=%s\n" % mstr config_file.write(line) # user may have options specific to certain yum plugins # add them to the file for x in repo.yumopts: config_file.write("%s=%s\n" % (x, repo.yumopts[x])) if x == "enabled": optenabled = True if x == "gpgcheck": optgpgcheck = True else: mstr = repo.mirror if mstr.startswith("/"): mstr = "file://%s" % mstr line = "baseurl=%s\n" % mstr if self.settings.http_port not in (80, '80'): http_server = "%s:%s" % (self.settings.server, self.settings.http_port) else: http_server = self.settings.server line = line.replace("@@server@@", http_server) config_file.write(line) config_proxy = None if repo.proxy == '<<inherit>>': config_proxy = self.settings.proxy_url_ext elif repo.proxy != '' and repo.proxy != '<<None>>': config_proxy = repo.proxy if config_proxy is not None: config_file.write("proxy=%s\n" % config_proxy) if 'exclude' in list(repo.yumopts.keys()): self.logger.debug("excluding: %s" % repo.yumopts['exclude']) config_file.write("exclude=%s\n" % repo.yumopts['exclude']) if not optenabled: config_file.write("enabled=1\n") config_file.write("priority=%s\n" % repo.priority) # FIXME: potentially might want a way to turn this on/off on a per-repo basis if not optgpgcheck: config_file.write("gpgcheck=0\n") # user may have options specific to certain yum plugins # add them to the file for x in repo.yumopts: config_file.write("%s=%s\n" % (x, repo.yumopts[x])) if x == "enabled": optenabled = True if x == "gpgcheck": optgpgcheck = True config_file.close() return fname
# ==================================================================================
[docs] def update_permissions(self, repo_path): """ Verifies that permissions and contexts after an rsync are as expected. Sending proper rsync flags should prevent the need for this, though this is largely a safeguard. """ # all_path = os.path.join(repo_path, "*") owner = "root:apache" (dist, _) = os_release() if dist == "suse": owner = "root:www" elif dist in ("debian", "ubuntu"): owner = "root:www-data" cmd1 = "chown -R " + owner + " %s" % repo_path utils.subprocess_call(self.logger, cmd1) cmd2 = "chmod -R 755 %s" % repo_path utils.subprocess_call(self.logger, cmd2)