#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
**Project Name:**      MakeHuman

**Product Home Page:** http://www.makehumancommunity.org/

**Github Code Home Page:**    https://github.com/makehumancommunity/

**Authors:**           Jonas Hauquier

**Copyright(c):**      MakeHuman Team 2001-2020

**Licensing:**         AGPL3

    This file is part of MakeHuman Community (www.makehumancommunity.org).

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.


Abstract
--------

Simple download script to fetch additional assets from ftp.
Syncs local and FTP content using FTP's modified dates and a locally cached
index file.
"""

## CONFIG #################################
ftpUrl = "download.tuxfamily.org"
ftpPath = "/makehuman/assets/"
defaultRepo = "base"

default_nodelete = False
###########################################

def version():
    return "1.2"


import os
import sys
import io
import shutil
from ftplib import FTP

sys.path = ["./lib"] + sys.path
from getpath import getSysDataPath, isSubPath, getSysPath

def downloadFromFTP(ftp, filePath, destination):
    fileSize = ftp.size(filePath)
    downloaded = [0]    # Is a list so we can pass and change it in an inner func
    global f
    if destination:
        f = io.open(destination, 'wb')
    else:
        f = ""

    def writeChunk(data):
        global f
        if destination:
            f.write(data)
        else:
            f = f+str(data)

        downloaded[0] += len(data)
        percentage = 100 * float(downloaded[0]) / fileSize
        if destination:
            sys.stdout.write("  Downloaded %d%% of %d bytes\r" % (percentage, fileSize))

        if percentage >= 100 and destination:
            sys.stdout.write('\n')

    ftp.retrbinary('RETR '+filePath, writeChunk)
    if destination:
        f.close()
    else:
        return f


def downloadFile(ftp, filePath, destination, fileProgress):
    filePath = filePath.replace('\\', '/')
    destination = destination.replace('\\', '/')
    if os.path.dirname(destination) and not os.path.isdir(os.path.dirname(destination)):
        os.makedirs(os.path.dirname(destination))

    #print ("[%d%% done] Downloading file %s to %s" % (fileProgress, url, filename))
    print("[%d%% done] Downloading file %s" % (fileProgress, os.path.basename(destination)))
    print("             %s ==> %s" % (filePath, destination))
    downloadFromFTP(ftp, filePath, destination)

def parseContentsFile(filename):
    import io
    f = io.open(filename, 'r', encoding="utf-8")
    fileData = f.read()
    contents = {}
    for l in fileData.split('\n'):
        if not l:
            continue
        if l.startswith('#') or l.startswith('//'):
            continue
        c=l.split()
        try:
            contents[c[0]] = int(c[1])
        except:
            contents[c[0]] = 0
    f.close()
    return contents

def writeContentsFile(filename, contents):
    import io
    f = io.open(filename, 'w', encoding="utf-8")
    for fPath, mtime in list(contents.items()):
        f.write("%s %s\n" % (fPath, mtime))
    f.close()

def getNewFiles(oldContents, newContents, destinationFolder):
    result = []
    for (filename, newTime) in list(newContents.items()):
        destFile = os.path.join(destinationFolder, filename.lstrip('/'))
        if not os.path.isfile(destFile):
            result.append(filename)
        elif filename in oldContents:
            oldTime = oldContents[filename]
            if newTime > oldTime:
                result.append(filename)
        else:
            result.append(filename)
    return result

def getRemovedFiles(oldContents, newContents, destinationFolder):
    toRemove = []
    for filename in list(oldContents.keys()):
        if filename not in newContents:
            destFile = os.path.join(destinationFolder, filename.lstrip('/'))
            if os.path.isfile(destFile):
                toRemove.append(filename)
    return toRemove

def getFTPContents(ftp):
    def walkFTP(ftp, subFraction = None, currentProgress = None):
        if subFraction is None:
            subFraction = 1.0
        if currentProgress is None:
            currentProgress = 0.0

        percentage = 100 * currentProgress
        sys.stdout.write("[%d%% done] Getting repository contents\r" % percentage)
        sys.stdout.flush()

        path = ftp.pwd()    # TODO relpath to root path
        contentsList = []
        directories = []
        s = ftp.retrlines('LIST', contentsList.append)

        for line in contentsList:
            if line.startswith('d'): # is a folder
                directories.append(line.split()[8])

        if len(directories):
            subFraction /= float(len(directories))

        filesList = [f for f in ftp.nlst() if f not in directories]
        if len(filesList):
            filesFraction = 1.0/len(filesList)
        result = []
        for fname in filesList:
            mtime = ftp.sendcmd('MDTM %s' % fname)
            mtime = int(mtime[3:].strip())
            #mtime = int(time.mktime(time.strptime(mtime[3:].strip(), '%Y%m%d%H%M%S')))
            fpath = os.path.join(path, fname)
            result.append( (fpath, mtime) )
            percentage += 100 * (filesFraction * subFraction)
            sys.stdout.write("[%d%% done] Getting repository contents\r" % percentage)
            sys.stdout.flush()

        for dir_ in directories:
            ftp.cwd(dir_.replace('\\', '/'))
            result.extend( walkFTP(ftp, subFraction, currentProgress) )
            ftp.cwd('..')
            currentProgress += subFraction

        return result

    print('Retrieving new repository content...')
    rootPath = ftp.pwd()
    contentsList = walkFTP(ftp)
    sys.stdout.write("[100% done] Getting repository contents\r")
    sys.stdout.write('\n')
    sys.stdout.flush()
    result = {}
    # make paths relative to rootpath
    for (p, mtime) in contentsList:
        result[os.path.relpath(p, rootPath)] = mtime

    return result

def downloadFromHTTP(url, destination):
    import urllib.request, urllib.parse, urllib.error

    def chunk_report(bytes_so_far, chunk_size, total_size):
        percentage = float(bytes_so_far) / total_size
        percentage = round(percentage*100, 2)
        sys.stdout.write("  Downloaded %d%% of %d bytes\r" % (percentage, total_size))
        sys.stdout.flush()

        if bytes_so_far >= total_size:
            sys.stdout.write('\n')

    def chunk_read(response, chunk_size=512, report_hook=None):
        total_size = response.info().getheader('Content-Length').strip()
        total_size = int(total_size)
        bytes_so_far = 0

        while True:
            chunk = response.read(chunk_size)
            f.write(chunk)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            if report_hook:
                report_hook(bytes_so_far, chunk_size, total_size)

        return bytes_so_far

    f = io.open(destination, 'wb')
    response = urllib.request.urlopen(url)
    if response.getcode() != 200:
        raise RuntimeError('Failed to download file %s (error code %s)' % (url, response.getcode()))
    chunk_read(response, report_hook=chunk_report)
    f.close()

def isArchived(ftp):
    return 'archive_url.txt' in ftp.nlst()

def getArgs():
    if len(sys.argv) < 2:
        return dict()

    import argparse    # requires python >= 2.7
    parser = argparse.ArgumentParser()

    # optional arguments
    parser.add_argument('-v', '--version', action='version', version=version())
    parser.add_argument("-d", "--nodelete", action="store_true", help="Don't delete old version when updating files")
    parser.add_argument('-y', '--yes-really-download-old-assets', action="store_true", help="Confirm you understand this means you download from tuxfamily")

    # optional positional arguments
    parser.add_argument("repository", default=defaultRepo, nargs='?', help="Alternative repository name to download from (optional)")

    argOptions = vars(parser.parse_args())
    return argOptions

def getVersion():
    """
    Version of MakeHuman software
    """
    import sys
    sys.path = ["."] + sys.path
    import makehuman
    return '.'.join([str(i) for i in makehuman.version[:2]])


if __name__ == '__main__':
    global DONTREMOVE

    args = getArgs()

    if not "yes_really_download_old_assets" in args or args["yes_really_download_old_assets"] is None or not args["yes_really_download_old_assets"]:
        print("\n!!! WARNING WARNING WARNING !!!\n")
        print("build_prepare.py downloads the old makehuman assets from tuxfamily, whereas")
        print("the current assets are stored on github. To download the current assets,")
        print("use download_assets_git.py instead.\n")
        print("If you understand the consequences, you can still use this old script and")
        print("get the old tuxfamily assets by running:\n")
        print("    python download_assets.py --yes-really-download-old-assets\n")
        print("!!! WARNING WARNING WARNING !!!\n")
        sys.exit(1)

    repo = args.get('repository', defaultRepo)
    for c in ['.', '/', '\\']:
        if c in repo:
            raise RuntimeError('Invalid argument for "repository", illegal character')
    DONTREMOVE = args.get('nodelete', default_nodelete)

    # Obtain MH version to download assets for
    version = getVersion()

    print('Refreshing assets from repository "%s" (version %s)' % (repo, version))

    ftpPath = os.path.join(ftpPath, version.lstrip('/'), repo.lstrip('/'))
    ftpPath = os.path.normpath(ftpPath)
    ## Use simple sync mechanism, maybe in the future we can use rsync over http?
    # Download contents list
    baseName = os.path.basename(ftpPath)
    contentsFile = getSysPath('%s_%s_contents.txt' % (baseName, version.replace('.','-')))
    if os.path.isfile(contentsFile):
        # Parse previous contents file
        oldContents = parseContentsFile(contentsFile)
        if len(oldContents) > 0 and len(str(oldContents[list(oldContents.keys())[0]])) < 5:
            # Ignore old style contents file
            oldContents = {}
    else:
        oldContents = {}

    # Setup FTP connection
    print("Connecting to FTP...")
    ftp = FTP(ftpUrl)
    ftp.login()
    ftp.cwd(ftpPath.replace('\\', '/'))

    # Verify if there is an archive reference URL
    if isArchived(ftp):
        print("Redirected to asset archive")
        archiveUrl = downloadFromFTP(ftp, 'archive_url.txt', None).strip()
        filename = os.path.basename(archiveUrl)
        zipDest = os.path.join(getSysPath(), filename)
        if os.path.exists(zipDest):
            print("Archive %s already exists, not downloading again." % zipDest)
            sys.exit()
        print("Downloading archive from HTTP (%s)" % archiveUrl)
        # Download and extract archive
        downloadFromHTTP(archiveUrl, zipDest)
        print("Extracting zip archive...")
        import zipfile
        zFile = zipfile.ZipFile(zipDest)
        zFile.extractall(getSysDataPath())

        print("All done.")
        sys.exit()

    # Get contents from FTP
    newContents = getFTPContents(ftp)

    destinationFolder = getSysDataPath()
    toDownload = getNewFiles(oldContents, newContents, destinationFolder)
    toRemove = getRemovedFiles(oldContents, newContents, destinationFolder)

    # Remove files removed on FTP
    for filePath in toRemove:
        filename = os.path.join(destinationFolder, filePath.lstrip('/'))

        if not isSubPath(filename, destinationFolder):
            raise RuntimeError("ERROR: File destinations are jailed inside the sys data path (%s), destination path (%s) tries to escape!" % (destinationFolder, filename))

        if DONTREMOVE:
            newFile = filename + '.removedasset'
            i = 0
            while os.path.isfile(newFile):
                newFile = filename + '.' + str(i) + '.removedasset'
                i = i+1
            shutil.move(filename, newFile)
            print("Moved removed file to %s (removed from FTP)" % newFile)
        else:
            print("Removing file %s (removed from FTP)" % filename)
            os.remove(filename)

    TOTAL_FILES = len(toDownload)

    fIdx = 0
    for filePath in toDownload:
        filename = os.path.join(destinationFolder, filePath.lstrip('/'))

        if not isSubPath(filename, destinationFolder):
            raise RuntimeError("ERROR: File destinations are jailed inside the sys data path (%s), destination path (%s) tries to escape!" % (destinationFolder, filename))

        if os.path.exists(filename) and DONTREMOVE:
            newFile = filename + '.oldasset'
            i = 0
            while os.path.exists(newFile):
                newFile = filename + '.' + str(i) + '.oldasset'
                i = i+1
            shutil.move(filename, newFile)
            print("Moved old version of updated file to %s" % newFile)

        fileProgress = round(100 * float(fIdx)/TOTAL_FILES, 2)
        downloadFile(ftp, filePath, filename, fileProgress)
        fIdx += 1

    ftp.close()

    writeContentsFile(contentsFile, newContents)

    print("All done.")

