handlemoodlesubmissions.py 8.21 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python

"""Extract student's submission files from Moodle assignment

Transfer PDF files from ZIP file containing all submissions of a Moodle
assignment into output folder with file names following exam scan
naming convention.

Attention: Contents in output folder will be overwritten in the following!
"""

__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
    "Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"


Amrita Deb's avatar
Amrita Deb committed
16
17
18
19
import sys  # get arguments from command line
import os  # path listing/manipulation/...
import time  # keep track of time
import argparse  # handle command line arguments
20
21
22
23
import shutil  # unzipping and copying files

from utils import moodle as moodle

Amrita Deb's avatar
Amrita Deb committed
24

25
26
def _make_parser():
    csv_parser = moodle.get_moodle_csv_parser()
Amrita Deb's avatar
Amrita Deb committed
27

28
29
    parser = argparse.ArgumentParser(
        parents=[csv_parser],
30
        description=__doc__, prog='handlemoodlesubmissions.py',
31
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
Amrita Deb's avatar
Amrita Deb committed
32

33
    parser.add_argument(
34
        "inzip", help="Input ZIP file or extracted folder.")
35
    parser.add_argument(
36
37
38
        "csv", help="Moodle grading sheet.")
    parser.add_argument(
        "outfolder", help="Output folder with PDFs.")
39

40
    parser.add_argument(
41
        "-f", "--filenameformat", default="{matnum}_{fullname[0]}",
42
        help="File name format. Available keywords: " +
Christian Rohlfing's avatar
Christian Rohlfing committed
43
44
        "{matnum}, {fullname}, {lastname}, {firstname}. " +
        "Default: '{matnum}_{fullname[0]}'")
45
    parser.add_argument(
46
        "-c", "--copyall", action='store_true',
47
48
        help="If set, copies all files (including multiple and non-PDF files)")
    parser.add_argument(
49
        "-a", "--appendoriginal", action='store_true',
50
51
52
53
        help="If set, appends original file name to new location's file name")
    parser.add_argument(
        "-d", "--dry", action='store_true', help="Flag for dry run.")
    parser.add_argument(
54
55
56
57
58
59
        "-t", "--tmp", default="./tmp", help="Temporary folder.")

    return parser


# Create argument parser with default values
Christian Rohlfing's avatar
Christian Rohlfing committed
60
_parser = _make_parser()
61
62
63
64
65
66
67
68
69
70
71
72
73


def main(args):
    """Main routine

    1) Files are extracted from zip file location eg: ./all_submissions.zip
        In case folder is given, extraction is skipped.
    2) Scan through extracted folder for PDF files.
        Only 1 PDF file/student is accepted.
    3) Matriculation number and last name are fetched from grading worksheet
    4) PDFs from extracted folder are renamed according to convention and
        placed in user provided outfolder
    """
Amrita Deb's avatar
Amrita Deb committed
74

75
    # Argument handling
Christian Rohlfing's avatar
Christian Rohlfing committed
76
    args = _parser.parse_args(args)
Amrita Deb's avatar
Amrita Deb committed
77
78
    inzip = args.inzip
    outfolder = args.outfolder
79
80
81
82
83
84
85
86
87
88
89
    sheet_csv = args.csv
    dry = args.dry
    csv_enc = args.csvenc
    csv_delim = args.csvdelim
    csv_quote = args.csvquote
    copy_all = args.copyall
    append_original_name = args.appendoriginal
    filenameformat = args.filenameformat
    tmp_folder = args.tmp
    extracted_folder = os.path.join(tmp_folder, "extracted_from_moodle")

Deb's avatar
Deb committed
90
91
92
93
    # Check folders
    if not os.path.exists(outfolder):
        os.makedirs(outfolder)

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    # Print status
    starttime = time.time()
    num_students = moodle.get_student_number(sheet_csv=sheet_csv,
                                             csv_enc=csv_enc)

    print('''Preparing for renaming of submission files.
Processing {} students
  '''.format(num_students))

    # Clean up and create temporary folder
    dryout = []
    if dry:
        print("Dry run\n")

    # Check whether zip or folder is given
    folder_instead_of_zip = False
    if not(inzip.lower().endswith(('.zip'))):
        if not(os.path.isdir(inzip)):
            raise Exception(
                "{zip} neither Zip file nor folder. Exiting."
                .format(zip=inzip))
        # Folder was given instead of Zip file
        extracted_folder = inzip
        folder_instead_of_zip = True
Amrita Deb's avatar
Amrita Deb committed
118
119

    else:
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
        # Extract
        print("Extracting files from {zip} ...".format(zip=inzip))
        if not dry:
            shutil.unpack_archive(inzip, extracted_folder)  # unzip file
        else:
            raise Exception("Dry run prevents me from unpacking the Zip file.")

    # List all extracted folders
    folders = os.listdir(extracted_folder)
    folders.sort()

    # There should never be more folders than entries in CSV file
    if len(folders) > num_students:
        raise Exception(
            ("More folders ({num_folders}) than "
             "students in CSV file ({num_students})")
            .format(num_folders=len(folders), num_students=num_students))

    # Parse grading infos from CSV file
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csv_delim,
                                csv_quote=csv_quote, csv_enc=csv_enc)

    # Collect non-default cases:
    # Student did not submit anything
    infos_no_submission = []
    # Student submitted more than one file
    infos_multi_files = []
    # Student submitted a non-PDF file
    infos_unsupported_files = []

    # Loop over grading info
    print("Copying submissions", sep=' ', end='', flush=True)
    for cnt, info in enumerate(infos):
        folder = moodle.submission_folder_name(info)

        if folder in folders:
            # Folder was found
            folderfull = os.path.join(extracted_folder, folder)
            files = os.listdir(folderfull)

            # Notify if folder empty
            if len(files) == 0:
                infos_no_submission.append(info)

            # Notify if more than one submission
            if len(files) > 1:
                infos_multi_files.append(info)

            # Iterate over all files within folder
            for file_cnt, file in enumerate(files):
                file_full = os.path.join(folderfull, file)

                # Create destination file name
                dest = filenameformat.format(
                    matnum=info['matnum'], fullname=info['fullname'],
                    lastname=info['lastname'], firstname=info['firstname'])

                # Add unique file ID (only for copy all)
                if copy_all > 0:
                    dest = dest + "_{:03d}".format(file_cnt)

                base, ext = os.path.splitext(file)
                # Add original file name
                if append_original_name:
                    dest = dest + "_" + base

                # Add extension
                dest = dest + ext
                dest_full = os.path.join(outfolder, dest)

                # Notify if non-PDF file
                is_pdf = file_full.lower().endswith('.pdf')
                if not is_pdf and \
                        info not in infos_unsupported_files:
                    infos_unsupported_files.append(info)

                # Copy either first PDF file or all files if copyall is active
                if (file_cnt == 0 and is_pdf) or copy_all:
                    if not dry:
                        shutil.copyfile(file_full, dest_full)
Amrita Deb's avatar
Amrita Deb committed
200
                    else:
201
                        dryout.append(
Christian Rohlfing's avatar
Christian Rohlfing committed
202
203
                            "- {old} -> {new}"
                            .format(old=os.path.join(folder, file), new=dest))
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
        else:
            # Notify if folder was not found
            infos_no_submission.append(info)

        # Print for-loop progress
        if not (cnt % max(1, round(num_students/10))):
            print(".", sep=' ', end='', flush=True)

    print("done.")

    # Report back special cases
    for report in [(infos_no_submission, "no files"),
                   (infos_multi_files, "multiple files"),
                   (infos_unsupported_files, "unsupported files")]:
        infos, reason = report
        if len(infos) > 0:
            lines = ["- {folder} ({matnum})"
                     .format(folder=moodle.submission_folder_name(_),
                             matnum=_['matnum'])
                     for _ in infos]
            lines.sort()
            print(
                "\nSubmissions of {reason}:\n{lines}"
                .format(reason=reason, lines="\n".join(lines)))

    # Dry run output
    if not dry:
        # Delete temporary folder
        if not folder_instead_of_zip:
            shutil.rmtree(extracted_folder)
    else:
        dryout.sort()
        print("\nDry run results:\n{}".format("\n".join(dryout)))

    # Print status
    endtime = time.time()
    print("Time taken: {:.2f}".format(endtime-starttime))
Amrita Deb's avatar
Amrita Deb committed
241
242
243


if __name__ == '__main__':
Amrita's avatar
Amrita committed
244
    main(sys.argv[1:])