wingin-installers/updatefiles.py

"""
This script can be used to update this repository with the latest information.
It performs the following actions:
- Updates the local clone of the repository
- Downloads the latest setup files from Appveyor
- Updates the README with the latest version info (commit ID and message)
- Uploads changes to the repository
"""
import os
import sys
import pickle
import requests
import json
from requests.exceptions import ConnectionError as ConnError
from subprocess import call


ETAGFILE = "etags"
ETAGS = {}  # type: dict

DOC = __doc__.replace("This script", f"The [{__file__}]({__file__}) script")


def load_etags():
    """
    Read in etags file and populates dictionary.
    """
    try:
        with open(ETAGFILE, "rb") as etagfile:
            ETAGS.update(pickle.load(etagfile))
    except FileNotFoundError:
        # print("--> No etags file found. Skipping load.")
        pass


def save_etags():
    """
    Save (potentially new) etags to file.
    """
    with open(ETAGFILE, "wb") as etagfile:
        pickle.dump(ETAGS, etagfile)


def download(url, fname=None):
    """
    Download a URL if necessary. If the URL's etag matches the existing one,
    the download is skipped.
    """
    if fname is None:
        fname = url.split("/")[-1]

    print("--> Downloading {} → {}".format(url, fname))
    try:
        req = requests.get(url, stream=True)
    except ConnError:
        print("Error while trying to download {}".format(url), file=sys.stderr)
        print("Skipping.", file=sys.stderr)
        return
    size = int(req.headers.get("content-length"))
    etag = req.headers.get("etag")
    oldet = ETAGS.get(url)
    if etag == oldet and os.path.exists(fname):
        fod_size = os.path.getsize(fname)
        if fod_size == size:
            print("File already downloaded. Skipping.", end="\n\n")
            return fname
    ETAGS[url] = etag
    prog = 0
    with open(fname, "wb") as dlfile:
        for chunk in req.iter_content(chunk_size=256):
            dlfile.write(chunk)
            prog += len(chunk)
            print("\r{:2.1f}%".format(prog / size * 100), end="", flush=True)
        print("\nDone!")
    print()
    return fname


def get_appveyor_info():
    # TODO: Check what happens when a build is in progress and so has no
    # available artifacts
    apiurl = "https://ci.appveyor.com/api/"
    account = "G-Node"
    project_name = "GinUI"

    url = os.path.join(apiurl, "projects", account, project_name)
    r = requests.get(url)

    projects = json.loads(r.text)
    build = projects["build"]
    info = dict()
    info["json"] = r.text
    info["commit"] = build["commitId"]
    info["message"] = build["message"]
    info["version"] = build["version"]
    dlurls = []
    for job in build["jobs"]:  # should just be one for this project
        if job["status"] == "success":
            artifacts_url = os.path.join(apiurl, "buildjobs", job["jobId"],
                                         "artifacts")
            r = requests.get(artifacts_url)
            artifacts = json.loads(r.text)
            for a in artifacts:
                dlurls.append(os.path.join(apiurl, "buildjobs", job["jobId"],
                                           "artifacts", a["fileName"]))
    info["artifacts"] = dlurls
    return info


def update_readme(info):
    print(f"Latest commit: {info['message']} [{info['commit']}]")
    commiturl = (f"https://github.com/G-Node/GinUI/commit/{info['commit']}")
    vertext = (f"\n## Version {info['version']}\n\n"
               "The current version of the setup files corresponds to:\n\n"
               f"- Commit ID: [{info['commit']}]({commiturl})\n"
               f"- Message:   {info['message']}\n")
    scriptdesc = f"\n## {__file__}\n{DOC}"

    with open("instructions.md") as instructfile:
        instructions = instructfile.read()
    with open("README.md", "w") as readme:
        readme.write(instructions)
        readme.write(vertext)
        readme.write(scriptdesc)


def update_verinfo(info):
    with open("version", "w") as verfile:
        verfile.write(info["version"])
    with open("build.json", "w") as jsonfile:
        jsonfile.write(info["json"])


def main():
    print("Updating local repository")
    call(["gin", "download", "--content"])
    print("Unlocking file content")
    call(["gin", "unlock", "."])
    load_etags()
    dlfiles = []
    avinfo = get_appveyor_info()
    artifacts = avinfo["artifacts"]
    print(f"Latest build message: {avinfo['message']}")
    if not len(artifacts):
        print("No artifacts in current build. Perhaps it is still running?")
        call(["gin", "lock", "."])
        sys.exit("aborting")
    print(f"Found {len(artifacts)} artifacts")
    for url in artifacts:
        dlfiles.append(download(url))
    save_etags()
    print("Updating README.md")
    update_readme(avinfo)
    update_verinfo(avinfo)
    print("Uploading changes")
    # at each run, we expect the following files to change
    changedfiles = ["README.md", "etags", "Setup.msi", "setup.exe", "version",
                    "build.json"]
    # any other changes (e.g., changes to this script) should be handled
    # manually
    call(["gin", "upload", *changedfiles])


if __name__ == "__main__":
    main()