path: root/vstf/vstf/common/rsync.py

##############################################################################
# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################


# from __future__ import nested_scopes

import os, os.path, shutil, glob, re, sys, getopt, stat, string

try:
    import win32file
except:
    win32file = None


class Cookie:
    def __init__(self):
        self.sink_root = ""
        self.target_root = ""
        self.quiet = 0
        self.recursive = 0
        self.relative = 0
        self.dry_run = 0
        self.time = 0
        self.update = 0
        self.cvs_ignore = 0
        self.ignore_time = 0
        self.delete = 0
        self.delete_excluded = 0
        self.delete_from_source = 0
        self.size_only = 0
        self.modify_window = 2
        self.existing = 0
        self.filters = []
        self.case_sensitivity = 0
        if os.name == "nt":
            self.case_sensitivity = re.I


def visit(cookie, dirname, names):
    """Copy files names from sink_root + (dirname - sink_root) to target_root + (dirname - sink_root)"""
    if os.path.split(cookie.sink_root)[
        1]:  # Should be tested with (C:\Cvs -> C:\)! (C:\Archives\MyDatas\UltraEdit -> C:\Archives\MyDatas) (Cvs -> "")! (Archives\MyDatas\UltraEdit -> Archives\MyDatas) (\Cvs -> \)! (\Archives\MyDatas\UltraEdit -> Archives\MyDatas)
        dirname = dirname[len(cookie.sink_root) + 1:]
    else:
        dirname = dirname[len(cookie.sink_root):]
    target_dir = os.path.join(cookie.target_root, dirname)
    if not os.path.isdir(target_dir):
        makeDir(cookie, target_dir)
    sink_dir = os.path.join(cookie.sink_root, dirname)

    filters = []
    if cookie.cvs_ignore:
        ignore = os.path.join(sink_dir, ".cvsignore")
        if os.path.isfile(ignore):
            filters = convertPatterns(ignore, "-")
    filters = filters + cookie.filters

    names_excluded = []
    if filters:
        # filter sink files (names):
        name_index = 0
        while name_index < len(names):
            name = names[name_index]
            path = os.path.join(dirname, name)
            path = convertPath(path)
            if os.path.isdir(os.path.join(sink_dir, name)):
                path = path + "/"
            for filter in filters:
                if re.search(filter[1], path, cookie.case_sensitivity):
                    if filter[0] == '-':
                        sink = os.path.join(sink_dir, name)
                        if cookie.delete_from_source:
                            if os.path.isfile(sink):
                                removeFile(cookie, sink)
                            elif os.path.isdir(sink):
                                removeDir(cookie, sink)
                            else:
                                logError("Sink %s is neither a file nor a folder (skip removal)" % sink)
                        names_excluded += [names[name_index]]
                        del (names[name_index])
                        name_index = name_index - 1
                        break
                    elif filter[0] == '+':
                        break
            name_index = name_index + 1

    if cookie.delete and os.path.isdir(target_dir):
        # Delete files and folder in target not present in filtered sink.
        for name in os.listdir(target_dir):
            if not cookie.delete_excluded and name in names_excluded:
                continue
            if not name in names:
                target = os.path.join(target_dir, name)
                if os.path.isfile(target):
                    removeFile(cookie, target)
                elif os.path.isdir(target):
                    removeDir(cookie, target)
                else:
                    pass

    for name in names:
        # Copy files and folder from sink to target.
        sink = os.path.join(sink_dir, name)
        # print sink
        target = os.path.join(target_dir, name)
        if os.path.exists(target):
            # When target already exit:
            if os.path.isfile(sink):
                if os.path.isfile(target):
                    # file-file
                    if shouldUpdate(cookie, sink, target):
                        updateFile(cookie, sink, target)
                elif os.path.isdir(target):
                    # file-folder
                    removeDir(cookie, target)
                    copyFile(cookie, sink, target)
                else:
                    # file-???
                    logError("Target %s is neither a file nor folder (skip update)" % sink)

            elif os.path.isdir(sink):
                if os.path.isfile(target):
                    # folder-file
                    removeFile(cookie, target)
                    makeDir(cookie, target)
            else:
                # ???-xxx
                logError("Sink %s is neither a file nor a folder (skip update)" % sink)

        elif not cookie.existing:
            # When target dont exist:
            if os.path.isfile(sink):
                # file
                copyFile(cookie, sink, target)
            elif os.path.isdir(sink):
                # folder
                makeDir(cookie, target)
            else:
                logError("Sink %s is neither a file nor a folder (skip update)" % sink)


def log(cookie, message):
    if not cookie.quiet:
        try:
            print message
        except UnicodeEncodeError:
            print message.encode("utf8")


def logError(message):
    try:
        sys.stderr.write(message + "\n")
    except UnicodeEncodeError:
        sys.stderr.write(message.encode("utf8") + "\n")


def shouldUpdate(cookie, sink, target):
    try:
        sink_st = os.stat(sink)
        sink_sz = sink_st.st_size
        sink_mt = sink_st.st_mtime
    except:
        logError("Fail to retrieve information about sink %s (skip update)" % sink)
        return 0

    try:
        target_st = os.stat(target)
        target_sz = target_st.st_size
        target_mt = target_st.st_mtime
    except:
        logError("Fail to retrieve information about target %s (skip update)" % target)
        return 0

    if cookie.update:
        return target_mt < sink_mt - cookie.modify_window

    if cookie.ignore_time:
        return 1

    if target_sz != sink_sz:
        return 1

    if cookie.size_only:
        return 0

    return abs(target_mt - sink_mt) > cookie.modify_window


def copyFile(cookie, sink, target):
    log(cookie, "copy: %s to: %s" % (sink, target))
    if not cookie.dry_run:
        try:
            shutil.copyfile(sink, target)
        except:
            logError("Fail to copy %s" % sink)

        if cookie.time:
            try:
                s = os.stat(sink)
                os.utime(target, (s.st_atime, s.st_mtime));
            except:
                logError("Fail to copy timestamp of %s" % sink)


def updateFile(cookie, sink, target):
    log(cookie, "update: %s to: %s" % (sink, target))
    if not cookie.dry_run:
        # Read only and hidden and system files can not be overridden.
        try:
            try:
                if win32file:
                    filemode = win32file.GetFileAttributesW(target)
                    win32file.SetFileAttributesW(target,
                                                 filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
                else:
                    os.chmod(target, stat.S_IWUSR)
            except:
                # logError("Fail to allow override of %s" % target)
                pass

            shutil.copyfile(sink, target)
            if cookie.time:
                try:
                    s = os.stat(sink)
                    os.utime(target, (s.st_atime, s.st_mtime));
                except:
                    logError(
                        "Fail to copy timestamp of %s" % sink)  # The utime api of the 2.3 version of python is not unicode compliant.
        except:
            logError("Fail to override %s" % sink)

        if win32file:
            win32file.SetFileAttributesW(target, filemode)


def prepareRemoveFile(path):
    if win32file:
        filemode = win32file.GetFileAttributesW(path)
        win32file.SetFileAttributesW(path,
                                     filemode & ~win32file.FILE_ATTRIBUTE_READONLY & ~win32file.FILE_ATTRIBUTE_HIDDEN & ~win32file.FILE_ATTRIBUTE_SYSTEM)
    else:
        os.chmod(path, stat.S_IWUSR)


def removeFile(cookie, target):
    # Read only files could not be deleted.
    log(cookie, "remove: %s" % target)
    if not cookie.dry_run:
        try:
            try:
                prepareRemoveFile(target)
            except:
                # logError("Fail to allow removal of %s" % target)
                pass

            os.remove(target)
        except:
            logError("Fail to remove %s" % target)


def makeDir(cookie, target):
    log(cookie, "make dir: %s" % target)
    if not cookie.dry_run:
        try:
            os.makedirs(target)
        except:
            logError("Fail to make dir %s" % target)


def visitForPrepareRemoveDir(arg, dirname, names):
    for name in names:
        path = os.path.join(dirname, name)
        prepareRemoveFile(path)


def prepareRemoveDir(path):
    prepareRemoveFile(path)
    os.path.walk(path, visitForPrepareRemoveDir, None)


def OnRemoveDirError(func, path, excinfo):
    logError("Fail to remove %s" % path)


def removeDir(cookie, target):
    # Read only directory could not be deleted.
    log(cookie, "remove dir: %s" % target)
    if not cookie.dry_run:
        prepareRemoveDir(target)
        try:
            shutil.rmtree(target, False, OnRemoveDirError)
        except:
            logError("Fail to remove dir %s" % target)


def convertPath(path):
    # Convert windows, mac path to unix version.
    separator = os.path.normpath("/")
    if separator != "/":
        path = re.sub(re.escape(separator), "/", path)

    # Help file, folder pattern to express that it should match the all file or folder name.
    path = "/" + path
    return path


def convertPattern(pattern, sign):
    """Convert a rsync pattern that match against a path to a filter that match against a converted path."""

    # Check for include vs exclude patterns.
    if pattern[:2] == "+ ":
        pattern = pattern[2:]
        sign = "+"
    elif pattern[:2] == "- ":
        pattern = pattern[2:]
        sign = "-"

    # Express windows, mac patterns in unix patterns (rsync.py extension).
    separator = os.path.normpath("/")
    if separator != "/":
        pattern = re.sub(re.escape(separator), "/", pattern)

    # If pattern contains '/' it should match from the start.
    temp = pattern
    if pattern[0] == "/":
        pattern = pattern[1:]
    if temp[-1] == "/":
        temp = temp[:-1]

    # Convert pattern rules: ** * ? to regexp rules.
    pattern = re.escape(pattern)
    pattern = string.replace(pattern, "\\?", ".")
    pattern = string.replace(pattern, "\\*\\*", ".*")
    pattern = string.replace(pattern, "\\*", "[^/]*")
    pattern = string.replace(pattern, "\\*", ".*")

    if "/" in temp:
        # If pattern contains '/' it should match from the start.
        pattern = "^\\/" + pattern
    else:
        # Else the pattern should match the all file or folder name.
        pattern = "\\/" + pattern

    if pattern[-2:] != "\\/" and pattern[-2:] != ".*":
        # File patterns should match also folders.
        pattern = pattern + "\\/?"

    # Pattern should match till the end.
    pattern = pattern + "$"
    return (sign, pattern)


def convertPatterns(path, sign):
    """Read the files for pattern and return a vector of filters"""
    filters = []
    f = open(path, "r")
    while 1:
        pattern = f.readline()
        if not pattern:
            break
        if pattern[-1] == "\n":
            pattern = pattern[:-1]

        if re.match("[\t ]*$", pattern):
            continue
        if pattern[0] == "#":
            continue
        filters = filters + [convertPattern(pattern, sign)]
    f.close()
    return filters


def printUsage():
    """Print the help string that should printed by rsync.py -h"""
    print "usage: rsync.py [options] source target"
    print """
 -q, --quiet              decrease verbosity
 -r, --recursive          recurse into directories
 -R, --relative           use relative path names
 -u, --update             update only (don't overwrite newer files)
 -t, --times              preserve times
 -n, --dry-run            show what would have been transferred
     --existing           only update files that already exist
     --delete             delete files that don't exist on the sending side
     --delete-excluded    also delete excluded files on the receiving side
     --delete-from-source delete excluded files on the receiving side
 -I, --ignore-times       don't exclude files that match length and time
     --size-only          only use file size when determining if a file should
                          be transferred
     --modify-window=NUM  timestamp window (seconds) for file match (default=2)
     --existing           only update existing target files or folders
 -C, --cvs-exclude        auto ignore files in the same way CVS does
     --exclude=PATTERN    exclude files matching PATTERN
     --exclude-from=FILE  exclude patterns listed in FILE
     --include=PATTERN    don't exclude files matching PATTERN
     --include-from=FILE  don't exclude patterns listed in FILE
     --version            print version number
 -h, --help               show this help screen

See http://www.vdesmedt.com/~vds2212/rsync.html for informations and updates.
Send an email to vivian@vdesmedt.com for comments and bug reports."""


def printVersion():
    print "rsync.py version 2.0.1"


def main(args):
    cookie = Cookie()

    opts, args = getopt.getopt(args, "qrRntuCIh",
                               ["quiet", "recursive", "relative", "dry-run", "time", "update", "cvs-ignore",
                                "ignore-times", "help", "delete", "delete-excluded", "delete-from-source", "existing",
                                "size-only", "modify-window=", "exclude=", "exclude-from=", "include=", "include-from=",
                                "version"])
    for o, v in opts:
        if o in ["-q", "--quiet"]:
            cookie.quiet = 1
        if o in ["-r", "--recursive"]:
            cookie.recursive = 1
        if o in ["-R", "--relative"]:
            cookie.relative = 1
        elif o in ["-n", "--dry-run"]:
            cookie.dry_run = 1
        elif o in ["-t", "--times",
                   "--time"]:  # --time is there to guaranty backward compatibility with previous buggy version.
            cookie.time = 1
        elif o in ["-u", "--update"]:
            cookie.update = 1
        elif o in ["-C", "--cvs-ignore"]:
            cookie.cvs_ignore = 1
        elif o in ["-I", "--ignore-time"]:
            cookie.ignore_time = 1
        elif o == "--delete":
            cookie.delete = 1
        elif o == "--delete-excluded":
            cookie.delete = 1
            cookie.delete_excluded = 1
        elif o == "--delete-from-source":
            cookie.delete_from_source = 1
        elif o == "--size-only":
            cookie.size_only = 1
        elif o == "--modify-window":
            cookie.modify_window = int(v)
        elif o == "--existing":
            cookie.existing = 1
        elif o == "--exclude":
            cookie.filters = cookie.filters + [convertPattern(v, "-")]
        elif o == "--exclude-from":
            cookie.filters = cookie.filters + convertPatterns(v, "-")
        elif o == "--include":
            cookie.filters = cookie.filters + [convertPattern(v, "+")]
        elif o == "--include-from":
            cookie.filters = cookie.filters + convertPatterns(v, "+")
        elif o == "--version":
            printVersion()
            return 0
        elif o in ["-h", "--help"]:
            printUsage()
            return 0

    if len(args) <= 1:
        printUsage()
        return 1

    # print cookie.filters

    target_root = args[1]
    try:  # In order to allow compatibility below 2.3.
        pass
        if os.path.__dict__.has_key("supports_unicode_filenames") and os.path.supports_unicode_filenames:
            target_root = unicode(target_root, sys.getfilesystemencoding())
    finally:
        cookie.target_root = target_root

    sinks = glob.glob(args[0])
    if not sinks:
        return 0

    sink_families = {}
    for sink in sinks:
        try:  # In order to allow compatibility below 2.3.
            if os.path.__dict__.has_key("supports_unicode_filenames") and os.path.supports_unicode_filenames:
                sink = unicode(sink, sys.getfilesystemencoding())
        except:
            pass
        sink_name = ""
        sink_root = sink
        sink_drive, sink_root = os.path.splitdrive(sink)
        while not sink_name:
            if sink_root == os.path.sep:
                sink_name = "."
                break
            sink_root, sink_name = os.path.split(sink_root)
        sink_root = sink_drive + sink_root
        if not sink_families.has_key(sink_root):
            sink_families[sink_root] = []
        sink_families[sink_root] = sink_families[sink_root] + [sink_name]

    for sink_root in sink_families.keys():
        if cookie.relative:
            cookie.sink_root = ""
        else:
            cookie.sink_root = sink_root

        global y  # In order to allow compatibility below 2.1 (nested scope where used before).
        y = sink_root
        files = filter(lambda x: os.path.isfile(os.path.join(y, x)), sink_families[sink_root])
        if files:
            visit(cookie, sink_root, files)

        # global y # In order to allow compatibility below 2.1 (nested scope where used before).
        y = sink_root
        folders = filter(lambda x: os.path.isdir(os.path.join(y, x)), sink_families[sink_root])
        for folder in folders:
            folder_path = os.path.join(sink_root, folder)
            if not cookie.recursive:
                visit(cookie, folder_path, os.listdir(folder_path))
            else:
                os.path.walk(folder_path, visit, cookie)
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))