preparemoodle.py

#!/usr/bin/env python

"""Prepares batch upload to Moodle's assignment module.

PDFs in folder 'in' are moved to a certain folder structure to be recognized
by moodle and finally zipped to 'outzip'.

Attention: Zip-archive 'outzip' will be overwritten in the following!
"""

__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
    "Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"


import os
import time  # keep track of time
import shutil  # copyfile, make_archive
import argparse  # argument parsing
import sys
import zipfile

import utils.moodle as moodle
import utils.matnum as matnum_utils


def zip_folders(base_folder, zip_file, size_limit):
    """Zip folders in base folder. If size limit is exceeded, create next zip
    file until all folders are zipped.

    Args:
        base_folder (str): path of base folder
        zip_file (str): path of zip file
        size_limit (int): size limit

    Returns:
        int: number of zip files
    """

    # Initialize
    zip_file_base = os.path.splitext(zip_file)[0]
    total_compress_size = 0
    zip_cnt = 0
    zf = None

    # Iterate over folders
    all_folders = os.listdir(base_folder)
    num_folders = len(all_folders)
    for cnt, folder in enumerate(all_folders):
        # Measure uncompressed folder path
        folder_path = os.path.join(base_folder, folder)
        folder_size = get_folder_size(folder_path)
        folder_size /= 1024**2  # conversion from bytes to MiB

        # If size_limit reached, create new zip file
        if total_compress_size + folder_size > size_limit or zf is None:
            # File name
            zip_cnt += 1
            if zf is not None:
                zf.close()  # Close previous zip file
                zip_file = "{zip}_{cnt}.zip".format(
                    zip=zip_file_base, cnt=zip_cnt)

            # Reset counters
            total_compress_size = 0
            file_cnt = 0

            # Open (new) zip file
            zf = zipfile.ZipFile(
                zip_file, mode='w', compression=zipfile.ZIP_DEFLATED)

        # Loop over files in current folder
        last_file_cnt = file_cnt
        for f in os.listdir(folder_path):
            # Add file to zip file
            zf.write(
                os.path.join(folder_path, f), arcname=os.path.join(folder, f))
            file_cnt += 1

        # Get compressed size of folder
        folder_compress_size = sum(
            [_.compress_size for _ in zf.filelist[last_file_cnt:]])
        folder_compress_size /= 1024**2  # conversion from bytes to MiB
        total_compress_size += folder_compress_size

        # Print for-loop progress
        if not (cnt % max(1, round(num_folders/10))):
            print(".", sep=' ', end='', flush=True)

    # Clean up
    zf.close()
    print("done.")

    return zip_cnt


def get_folder_size(path):
    """Get size in bytes of folder

    Args:
        path (str): path of folder

    Returns:
        int: number of bytes
    """
    total = 0
    for entry in os.scandir(path):
        if entry.is_file():
            size = entry.stat().st_size
        elif entry.is_dir():
            size = get_folder_size(entry.path)

        total += size

    return total


def sanity_check(matnums_csv, matnums_folder):
    """Check two cases for sanity:
    - Are there PDF files with no corresponding CSV entries?
    - Are there CSV entries with no provided PDF file?

    Args:
        matnums_csv (list): Matnums of all CSV entries
        matnums_folder (list): Matnums of all provided PDF files
    """

    # PDF files with no entry in CSV:
    notfoundcsv = list(set(matnums_folder).difference(matnums_csv))

    # Entries in CSV without PDF file
    notfoundpdf = list(set(matnums_csv).difference(matnums_folder))

    # Report back
    if len(notfoundcsv) > 0:
        print('''Warning: Following {} matnums have PDFs but no entry in CSV:
            {}'''.format(len(notfoundcsv), ", ".join(notfoundcsv)))

    if len(notfoundpdf) > 0:
        print('''Warning: Following {} matnums have CSV entries but no PDF:
            {}'''.format(len(notfoundpdf), ", ".join(notfoundpdf)))

    print("Done.\n")

    return notfoundcsv, notfoundpdf


def _make_parser():
    csv_parser = moodle.get_moodle_csv_parser()
    parser = argparse.ArgumentParser(
        parents=[csv_parser],
        description=__doc__,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument(
        "infolder", help="Input folder with PDFs.")
    parser.add_argument(
        "csv", help="Moodle grading sheet.")
    parser.add_argument(
        "outzip", help="Zip archive with feedback files.")

    parser.add_argument(
        "-d", "--dry", action='store_true', help="Flag for dry run.")
    parser.add_argument(
        "-t", "--tmp", default="./tmp", help="Temporary folder.")
    parser.add_argument(
        "--nowarn", action='store_true', help="Disables warnings.")
    parser.add_argument(
        "--moodleuploadlimit", default="250",
        help="Moodle upload limit in MiB.")

    return parser


# Create argument parser with default values
parser = _make_parser()
__doc__ += parser.format_help()


def main(args):
    """Main routine
    """

    # Parse input arguments
    args = parser.parse_args(args)
    infolder = args.infolder
    sheet_csv = args.csv
    outzip = args.outzip
    tmp_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle")
    dry = args.dry
    no_warn = args.nowarn
    csv_delim = args.csvdelim
    csv_quote = args.csvquote
    csv_enc = args.csvenc
    size_limit = int(args.moodleuploadlimit)  # Moodle upload size limit in MiB

    # Print status
    starttime = time.time()
    num_students = moodle.get_student_number(sheet_csv=sheet_csv,
                                             csv_enc=csv_enc)

    print('''Preparing for moodle upload
Processing {} students'''.format(num_students))

    # Clean up and create temporary folder
    dryout = []
    if dry:
        print("Dry run")
    else:
        # Remove zip file
        if os.path.exists(outzip):
            os.remove(outzip)

        # Create temporary folder within given temporary directory
        if os.path.isdir(tmp_folder):
            shutil.rmtree(tmp_folder)
        os.mkdir(tmp_folder)

    # Parse input folder
    # Only PDF files are considered with first digits
    # containing matriculation number
    matnums_folder = []
    allfiles = os.listdir(infolder)
    allfiles.sort()
    allpdfs = []
    for f in allfiles:
        if f.lower().endswith('.pdf') and matnum_utils.starts_with_matnum(f):
            allpdfs.append(f)
            matnums_folder.append(matnum_utils.get_matnum(f))

    # Parse grading infos from CSV file
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csv_delim,
                                csv_quote=csv_quote, csv_enc=csv_enc)

    # Loop over grading infos
    num_found_pdfs = 0
    matnums_csv = []
    moodleids = []
    if no_warn:
        print("Start copying", sep=' ', end='', flush=True)
    else:
        print("Start copying")
    for cnt, info in enumerate(infos):
        # Copy PDF files
        # Find all PDFs starting with matriculation number, e.g.
        # '123456_Lastname_sheet.pdf' and '123456_Lastname_exam.pdf'
        # If pdf files for current student exists, create a directory and
        # copy the pdf files to it. The resulting directories can be
        # uploaded to Moodle
        matnum = info['matnum']
        matnums_csv.append(matnum)
        moodleid = info['moodleid']
        moodleids.append(moodleid)

        pdfs_student = [_ for _ in allpdfs
                        if matnum == matnum_utils.get_matnum(_)]
        if len(pdfs_student) > 0:  # Found at least one pdf
            num_found_pdfs += len(pdfs_student)

            # Prepare submission folder
            folder = moodle.submission_folder_name(info)
            longfolder = os.path.join(tmp_folder, folder)

            # Create folder
            if not dry:
                os.mkdir(longfolder)

            # Copy all files to folder
            for pdffile in pdfs_student:
                longpdffile = os.path.join(infolder, pdffile)
                longpdffiledest = os.path.join(longfolder, pdffile)
                if not dry:
                    shutil.copyfile(longpdffile, longpdffiledest)
                else:
                    dryout.append(
                        "- {old} -> {new}"
                        .format(
                            old=pdffile, new=os.path.join(folder, pdffile)))

        elif not no_warn:  # No PDF found
            print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
                  .format(matnum=matnum, id=moodleid, name=info['fullname']))

        # Print for-loop progress
        if no_warn and not (cnt % max(1, round(num_students/10))):
            print(".", sep=' ', end='', flush=True)

    # Print results
    print("done.")
    print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
          .format(num_pdf=num_found_pdfs, num_csv=num_students))

    # Sanity check:
    # Check for PDFs not reflected in CSV (student not registered in Moodle)
    sanity_check(matnums_csv, matnums_folder)

    # Zip
    if not dry:
        print("Zipping")
        zip_cnt = zip_folders(
            base_folder=tmp_folder, zip_file=outzip, size_limit=size_limit)

        # Remove temporary folder
        shutil.rmtree(tmp_folder)

        # Print status
        print("{cnt} zip archives are stored ({zip}*)"
              .format(cnt=zip_cnt, zip=os.path.splitext(outzip)[0]))

    # Print dry run results
    else:
        dryout.sort()
        print("\nDry run results:\n{}".format("\n".join(dryout)))

    # Print status
    endtime = time.time()
    print("Time taken: {:.2f}".format(endtime-starttime))


# Main routine
if __name__ == '__main__':
    main(sys.argv[1:])