preparemoodle.py 6.83 KB
Newer Older
1
2
#!/usr/bin/env python

3
4
import os
import time
5
import shutil  # copyfile, make_archive
6
import argparse  # argument parsing
7
import sys
8
9

import utils.moodle as moodle
10
import utils.matnum as matnum_utils
11
12


13
14
15
16
def sanity_check(matnums_csv, matnums_folder):
    """Check two cases for sanity:
    - Are there PDF files with no corresponding CSV entries?
    - Are there CSV entries with no provided PDF file?
17
18

    Args:
19
20
        matnums_csv (list): Matnums of all CSV entries
        matnums_folder (list): Matnums of all provided PDF files
21
22
    """

23
24
    # PDF files with no entry in CSV:
    notfoundcsv = list(set(matnums_folder).difference(matnums_csv))
25

26
27
    # Entries in CSV without PDF file
    notfoundpdf = list(set(matnums_csv).difference(matnums_folder))
28
29

    # Report back
30
31
32
33
34
35
36
    if len(notfoundcsv) > 0:
        print('''Warning: Following {} matnums have PDFs but no entry in CSV:
            {}'''.format(len(notfoundcsv), ", ".join(notfoundcsv)))

    if len(notfoundpdf) > 0:
        print('''Warning: Following {} matnums have CSV entries but no PDF:
            {}'''.format(len(notfoundpdf), ", ".join(notfoundpdf)))
37
38
39

    print("Done.\n")

40
41
    return notfoundcsv, notfoundpdf

42
43
44
45
46
47
48
49

def main(args):
    """Main routine
    """

    # Parse input arguments
    parser = argparse.ArgumentParser(description='''
    prepares batch upload to Moodle via assignment module.
50
51
    PDFs in folder 'in' are moved to folder 'tmp' with a certain folder
    structure and finally zipped to 'out'.
52
53
54
    Attention: zip-archive 'out' will be overwritten in the following!

    ''')
Deb's avatar
Deb committed
55
56
    parser.add_argument("infolder", help="Input folder with PDFs.")
    parser.add_argument("csv", help="Moodle grading sheet.")
57
58
59
60
61
62
63
64
    parser.add_argument(
        "--csvdelim", default=",", help="CSV delimiter. Default: ','")
    parser.add_argument(
        "--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
    parser.add_argument(
        "--csvenc", default="utf-8", help="CSV encoding scheme. " +
        "Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
        "Default: 'utf-8'")
Deb's avatar
Deb committed
65
    parser.add_argument("outzip", help="Zip archive.")
66
67
68
69
70
71
    parser.add_argument(
        "-d", "--dry", action='store_true', help="Flag for dry run.")
    parser.add_argument(
        "-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
    parser.add_argument(
        "--nowarn", action='store_true', help="Disables warnings")
72
73
74

    args = parser.parse_args(args)
    infolder = args.infolder
75
    sheet_csv = args.csv
76
    outzip = args.outzip
77
    tmp_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle")
78
    dry = args.dry
79
80
81
82
    no_warn = args.nowarn
    csv_delim = args.csvdelim
    csv_quote = args.csvquote
    csv_enc = args.csvenc
83

84
    # Print status
85
    starttime = time.time()
86
87
    num_students = moodle.get_student_number(sheet_csv=sheet_csv,
                                             csv_enc=csv_enc)
88
89

    print('''Preparing for moodle upload
Christian Rohlfing's avatar
Christian Rohlfing committed
90
Processing {} students'''.format(num_students))
91

92
    # Clean up and create temporary folder
Christian Rohlfing's avatar
Christian Rohlfing committed
93
    dryout = []
94
    if dry:
Christian Rohlfing's avatar
Christian Rohlfing committed
95
        print("Dry run")
96
97
98
99
100
101
    else:
        # Remove zip file
        if os.path.exists(outzip):
            os.remove(outzip)

        # Create temporary folder within given temporary directory
102
103
        if not os.path.isdir(tmp_folder):
            os.mkdir(tmp_folder)
104

105
106
107
108
109
110
111
    # Parse input folder
    # Only PDF files are considered with first digits
    # containing matriculation number
    matnums_folder = []
    allfiles = os.listdir(infolder)
    allfiles.sort()
    allpdfs = []
Deb's avatar
Deb committed
112
113
114
    if (len(allfiles) == 0):
        print(""" There are no PDFs in the given directory. Exiting now.""")
        return
115
116
117
118
119
120
    for f in allfiles:
        if f.lower().endswith('.pdf') and matnum_utils.starts_with_matnum(f):
            allpdfs.append(f)
            matnums_folder.append(matnum_utils.get_matnum(f))

    # Parse grading infos from CSV file
121
122
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csv_delim,
                                csv_quote=csv_quote, csv_enc=csv_enc)
123
124

    # Loop over grading infos
125
    num_found_pdfs = 0
126
127
    matnums_csv = []
    moodleids = []
Christian Rohlfing's avatar
Christian Rohlfing committed
128
129
130
131
    if no_warn:
        print("Start processing", sep=' ', end='', flush=True)
    else:
        print("Start processing")
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
    for cnt, info in enumerate(infos):
        # Copy PDF files
        # Find all PDFs starting with matriculation number, e.g.
        # '123456_Lastname_sheet.pdf' and '123456_Lastname_exam.pdf'
        # If pdf files for current student exists, create a directory and
        # copy the pdf files to it. The resulting directories can be
        # uploaded to Moodle
        matnum = info['matnum']
        matnums_csv.append(matnum)
        moodleid = info['moodleid']
        moodleids.append(moodleid)

        pdfs_student = [_ for _ in allpdfs
                        if matnum == matnum_utils.get_matnum(_)]
        if len(pdfs_student) > 0:  # Found at least one pdf
147
            num_found_pdfs += len(pdfs_student)
148
149
150

            # Prepare submission folder
            folder = moodle.submission_folder_name(info)
151
            longfolder = os.path.join(tmp_folder, folder)
152
153
154
155
156
157
158
159
160

            # Create folder
            if not dry:
                os.mkdir(longfolder)

            # Copy all files to folder
            for pdffile in pdfs_student:
                longpdffile = os.path.join(infolder, pdffile)
                longpdffiledest = os.path.join(longfolder, pdffile)
161
                if not dry:
162
163
                    shutil.copyfile(longpdffile, longpdffiledest)
                else:
Christian Rohlfing's avatar
Christian Rohlfing committed
164
165
166
                    dryout.append(
                        "- {old} -> {new}"
                        .format(old=pdffile, new=os.path.join(folder, pdffile)))
167

168
        elif not no_warn:  # No PDF found
169
170
171
172
            print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
                  .format(matnum=matnum, id=moodleid, name=info['fullname']))

        # Print for-loop progress
Christian Rohlfing's avatar
Christian Rohlfing committed
173
        if no_warn and not (cnt % max(1, round(num_students/10))):
174
            print(".", sep=' ', end='', flush=True)
175
176

    # Print results
Christian Rohlfing's avatar
Christian Rohlfing committed
177
    print("done.")
178
179
    print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
          .format(num_pdf=num_found_pdfs, num_csv=num_students))
Christian Rohlfing's avatar
Christian Rohlfing committed
180
    
181
182
183

    # Sanity check:
    # Check for PDFs not reflected in CSV (student not registered in Moodle)
184
    sanity_check(matnums_csv, matnums_folder)
185

186
    # Zip
187
188
189
    if not dry:
        # Zip
        print("Zipping")
190
        shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmp_folder)
191
        print('Zip archive is stored at {}'.format(outzip))
192
193

        # Delete temporary folder
194
        shutil.rmtree(tmp_folder)
195

196
    # Print dry run results
197
    else:
Christian Rohlfing's avatar
Christian Rohlfing committed
198
199
        dryout.sort()
        print("\nDry run results:\n{}".format("\n".join(dryout)))
200

201
    # Print status
202
203
204
    endtime = time.time()
    print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))
205
206


207
# Main routine
208
if __name__ == '__main__':
209
    main(sys.argv[1:])