preparemoodle.py

#!/usr/bin/env python

import os
import time
import shutil  # copyfile, make_archive
import argparse  # argument parsing
import sys

import utils.moodle as moodle
import utils.matnum as matnum_utils

import zipfile


def bytesto(bytes, to, bsize=1024):
    """convert bytes to megabytes, etc.
       sample code:
           print('mb= ' + str(bytesto(314575262000000, 'm')))
       sample output: 
           mb= 300002347.946
    """

    a = {'k' : 1, 'm': 2, 'g' : 3, 't' : 4, 'p' : 5, 'e' : 6 }
    r = float(bytes)
    for i in range(a[to]):
        r = r / bsize

    return(round(r,2))

def sanity_check(matnums_csv, matnums_folder):
    """Check two cases for sanity:
    - Are there PDF files with no corresponding CSV entries?
    - Are there CSV entries with no provided PDF file?

    Args:
        matnums_csv (list): Matnums of all CSV entries
        matnums_folder (list): Matnums of all provided PDF files
    """

    # PDF files with no entry in CSV:
    notfoundcsv = list(set(matnums_folder).difference(matnums_csv))

    # Entries in CSV without PDF file
    notfoundpdf = list(set(matnums_csv).difference(matnums_folder))

    # Report back
    if len(notfoundcsv) > 0:
        print('''Warning: Following {} matnums have PDFs but no entry in CSV:
            {}'''.format(len(notfoundcsv), ", ".join(notfoundcsv)))

    if len(notfoundpdf) > 0:
        print('''Warning: Following {} matnums have CSV entries but no PDF:
            {}'''.format(len(notfoundpdf), ", ".join(notfoundpdf)))

    print("Done.\n")

    return notfoundcsv, notfoundpdf


def main(args):
    """Main routine
    """

    # Parse input arguments
    parser = argparse.ArgumentParser(description='''
    prepares batch upload to Moodle via assignment module.
    PDFs in folder 'in' are moved to folder 'tmp' with a certain folder
    structure and finally zipped to 'out'.
    Attention: zip-archive 'out' will be overwritten in the following!

    ''')
    parser.add_argument("infolder", help="Input folder with PDFs.")
    parser.add_argument("csv", help="Moodle grading sheet.")
    parser.add_argument(
        "--csvdelim", default=",", help="CSV delimiter. Default: ','")
    parser.add_argument(
        "--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
    parser.add_argument(
        "--csvenc", default="utf-8", help="CSV encoding scheme. " +
        "Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
        "Default: 'utf-8'")
    parser.add_argument("outzip", help="Zip archive.")
    parser.add_argument(
        "-d", "--dry", action='store_true', help="Flag for dry run.")
    parser.add_argument(
        "-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
    parser.add_argument(
        "--nowarn", action='store_true', help="Disables warnings")

    args = parser.parse_args(args)
    infolder = args.infolder
    sheet_csv = args.csv
    outzip = args.outzip
    tmp_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle")
    dry = args.dry
    no_warn = args.nowarn
    csv_delim = args.csvdelim
    csv_quote = args.csvquote
    csv_enc = args.csvenc

    # Print status
    starttime = time.time()
    num_students = moodle.get_student_number(sheet_csv=sheet_csv,
                                             csv_enc=csv_enc)

    print('''Preparing for moodle upload
Processing {} students'''.format(num_students))

    # Clean up and create temporary folder
    dryout = []
    if dry:
        print("Dry run")
    else:
        # Remove zip file
        if os.path.exists(outzip):
            os.remove(outzip)

        # Create temporary folder within given temporary directory
        if not os.path.isdir(tmp_folder):
            os.mkdir(tmp_folder)

    # Parse input folder
    # Only PDF files are considered with first digits
    # containing matriculation number
    matnums_folder = []
    allfiles = os.listdir(infolder)
    allfiles.sort()
    allpdfs = []
    if (len(allfiles) == 0):
        print(""" There are no PDFs in the given directory. Exiting now.""")
        return
    for f in allfiles:
        if f.lower().endswith('.pdf') and matnum_utils.starts_with_matnum(f):
            allpdfs.append(f)
            matnums_folder.append(matnum_utils.get_matnum(f))

    # Parse grading infos from CSV file
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csv_delim,
                                csv_quote=csv_quote, csv_enc=csv_enc)

    # Loop over grading infos
    num_found_pdfs = 0
    matnums_csv = []
    moodleids = []
    if no_warn:
        print("Start processing", sep=' ', end='', flush=True)
    else:
        print("Start processing")
    for cnt, info in enumerate(infos):
        # Copy PDF files
        # Find all PDFs starting with matriculation number, e.g.
        # '123456_Lastname_sheet.pdf' and '123456_Lastname_exam.pdf'
        # If pdf files for current student exists, create a directory and
        # copy the pdf files to it. The resulting directories can be
        # uploaded to Moodle
        matnum = info['matnum']
        matnums_csv.append(matnum)
        moodleid = info['moodleid']
        moodleids.append(moodleid)

        pdfs_student = [_ for _ in allpdfs
                        if matnum == matnum_utils.get_matnum(_)]
        if len(pdfs_student) > 0:  # Found at least one pdf
            num_found_pdfs += len(pdfs_student)

            # Prepare submission folder
            folder = moodle.submission_folder_name(info)
            longfolder = os.path.join(tmp_folder, folder)
            
            # Create folder
            if not dry:
                os.mkdir(longfolder)

            # Copy all files to folder
            for pdffile in pdfs_student:
                longpdffile = os.path.join(infolder, pdffile)
                longpdffiledest = os.path.join(longfolder, pdffile)
                if not dry:
                    shutil.copyfile(longpdffile, longpdffiledest)
                else:
                    dryout.append(
                        "- {old} -> {new}"
                        .format(old=pdffile, new=os.path.join(folder, pdffile)))

        elif not no_warn:  # No PDF found
            print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
                  .format(matnum=matnum, id=moodleid, name=info['fullname']))

        # Print for-loop progress
        if no_warn and not (cnt % max(1, round(num_students/10))):
            print(".", sep=' ', end='', flush=True)

    # Print results
    print("done.")
    print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
          .format(num_pdf=num_found_pdfs, num_csv=num_students))
    

    # Sanity check:
    # Check for PDFs not reflected in CSV (student not registered in Moodle)
    sanity_check(matnums_csv, matnums_folder)

    # Zip
    if not dry:
        foldersize = 0
        count = 1
        z = zipfile.ZipFile(os.path.splitext(outzip)[0]+str(count)+".zip", "w")
        for dirpath, dirnames, filenames in os.walk(tmp_folder):
            for file in filenames:
                file_path = os.path.join(dirpath, file)
                
                if not os.path.islink(file_path):
                    foldersize += os.path.getsize(file_path)
                    if bytesto(foldersize,'m') < 250:
                        z.write(file_path,os.path.join(os.path.relpath(file_path, tmp_folder),file))
                        os.remove(file_path)
                    else:
                        print("Preparing zip file "+str(count))
                        z.close()
                        print('Zip archive is stored at {}'.format(os.path.splitext(outzip)[0]+str(count)+".zip"))
                        count+=1
                        foldersize = 0
                        z = zipfile.ZipFile(os.path.splitext(outzip)[0]+str(count)+".zip", "w")
                        z.write(file_path,os.path.join(os.path.relpath(file_path, tmp_folder),file))
                        os.remove(file_path)

        # Delete temporary folder
        shutil.rmtree(tmp_folder)

    # Print dry run results
    else:
        dryout.sort()
        print("\nDry run results:\n{}".format("\n".join(dryout)))

    # Print status
    endtime = time.time()
    print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))


# Main routine
if __name__ == '__main__':
    main(sys.argv[1:])