preparemoodle.py 6.96 KB
Newer Older
1
2
#!/usr/bin/env python

3
4
import os
import time
5
import shutil  # copyfile, make_archive
6
import argparse  # argument parsing
7
import sys
8
9

import utils.moodle as moodle
10
import utils.matnum as matnum_utils
11
12


13
14
15
16
def sanity_check(matnums_csv, matnums_folder):
    """Check two cases for sanity:
    - Are there PDF files with no corresponding CSV entries?
    - Are there CSV entries with no provided PDF file?
17
18

    Args:
19
20
        matnums_csv (list): Matnums of all CSV entries
        matnums_folder (list): Matnums of all provided PDF files
21
22
    """

23
24
    # PDF files with no entry in CSV:
    notfoundcsv = list(set(matnums_folder).difference(matnums_csv))
25

26
27
    # Entries in CSV without PDF file
    notfoundpdf = list(set(matnums_csv).difference(matnums_folder))
28
29

    # Report back
30
31
32
33
34
35
36
    if len(notfoundcsv) > 0:
        print('''Warning: Following {} matnums have PDFs but no entry in CSV:
            {}'''.format(len(notfoundcsv), ", ".join(notfoundcsv)))

    if len(notfoundpdf) > 0:
        print('''Warning: Following {} matnums have CSV entries but no PDF:
            {}'''.format(len(notfoundpdf), ", ".join(notfoundpdf)))
37
38
39

    print("Done.\n")

40
41
    return notfoundcsv, notfoundpdf

42
43
44
45
46
47
48
49

def main(args):
    """Main routine
    """

    # Parse input arguments
    parser = argparse.ArgumentParser(description='''
    prepares batch upload to Moodle via assignment module.
50
51
    PDFs in folder 'in' are moved to folder 'tmp' with a certain folder
    structure and finally zipped to 'out'.
52
53
54
    Attention: zip-archive 'out' will be overwritten in the following!

    ''')
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    parser.add_argument(
        "-i", "--infolder", default="./pdfs_encrypted",
        help="Input folder with PDFs. Default: ./pdfs_encrypted")
    parser.add_argument(
        "-c", "--csv", default="./Bewertungen.csv",
        help="Moodle grading sheet. Default: ./Bewertungen.csv")
    parser.add_argument(
        "--csvdelim", default=",", help="CSV delimiter. Default: ','")
    parser.add_argument(
        "--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
    parser.add_argument(
        "--csvenc", default="utf-8", help="CSV encoding scheme. " +
        "Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
        "Default: 'utf-8'")
    parser.add_argument(
        "-o", "--outzip", default="./moodle_feedbacks.zip",
        help="Zip archive. Default: ./moodle_feedbacks.zip")
    parser.add_argument(
        "-d", "--dry", action='store_true', help="Flag for dry run.")
    parser.add_argument(
        "-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
    parser.add_argument(
        "--nowarn", action='store_true', help="Disables warnings")
78
79
80

    args = parser.parse_args(args)
    infolder = args.infolder
81
    sheet_csv = args.csv
82
    outzip = args.outzip
83
    tmp_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle")
84
    dry = args.dry
85
86
87
88
    no_warn = args.nowarn
    csv_delim = args.csvdelim
    csv_quote = args.csvquote
    csv_enc = args.csvenc
89

90
    # Print status
91
    starttime = time.time()
92
93
    num_students = moodle.get_student_number(sheet_csv=sheet_csv,
                                             csv_enc=csv_enc)
94
95

    print('''Preparing for moodle upload
Christian Rohlfing's avatar
Christian Rohlfing committed
96
Processing {} students'''.format(num_students))
97

98
    # Clean up and create temporary folder
Christian Rohlfing's avatar
Christian Rohlfing committed
99
    dryout = []
100
    if dry:
Christian Rohlfing's avatar
Christian Rohlfing committed
101
        print("Dry run")
102
103
104
105
106
107
    else:
        # Remove zip file
        if os.path.exists(outzip):
            os.remove(outzip)

        # Create temporary folder within given temporary directory
108
109
        if not os.path.isdir(tmp_folder):
            os.mkdir(tmp_folder)
110

111
112
113
114
115
116
117
118
119
120
121
122
123
    # Parse input folder
    # Only PDF files are considered with first digits
    # containing matriculation number
    matnums_folder = []
    allfiles = os.listdir(infolder)
    allfiles.sort()
    allpdfs = []
    for f in allfiles:
        if f.lower().endswith('.pdf') and matnum_utils.starts_with_matnum(f):
            allpdfs.append(f)
            matnums_folder.append(matnum_utils.get_matnum(f))

    # Parse grading infos from CSV file
124
125
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csv_delim,
                                csv_quote=csv_quote, csv_enc=csv_enc)
126
127

    # Loop over grading infos
128
    num_found_pdfs = 0
129
130
    matnums_csv = []
    moodleids = []
Christian Rohlfing's avatar
Christian Rohlfing committed
131
132
133
134
    if no_warn:
        print("Start processing", sep=' ', end='', flush=True)
    else:
        print("Start processing")
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    for cnt, info in enumerate(infos):
        # Copy PDF files
        # Find all PDFs starting with matriculation number, e.g.
        # '123456_Lastname_sheet.pdf' and '123456_Lastname_exam.pdf'
        # If pdf files for current student exists, create a directory and
        # copy the pdf files to it. The resulting directories can be
        # uploaded to Moodle
        matnum = info['matnum']
        matnums_csv.append(matnum)
        moodleid = info['moodleid']
        moodleids.append(moodleid)

        pdfs_student = [_ for _ in allpdfs
                        if matnum == matnum_utils.get_matnum(_)]
        if len(pdfs_student) > 0:  # Found at least one pdf
150
            num_found_pdfs += len(pdfs_student)
151
152
153

            # Prepare submission folder
            folder = moodle.submission_folder_name(info)
154
            longfolder = os.path.join(tmp_folder, folder)
155
156
157
158
159
160
161
162
163

            # Create folder
            if not dry:
                os.mkdir(longfolder)

            # Copy all files to folder
            for pdffile in pdfs_student:
                longpdffile = os.path.join(infolder, pdffile)
                longpdffiledest = os.path.join(longfolder, pdffile)
164
                if not dry:
165
166
                    shutil.copyfile(longpdffile, longpdffiledest)
                else:
Christian Rohlfing's avatar
Christian Rohlfing committed
167
168
169
                    dryout.append(
                        "- {old} -> {new}"
                        .format(old=pdffile, new=os.path.join(folder, pdffile)))
170

171
        elif not no_warn:  # No PDF found
172
173
174
175
            print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
                  .format(matnum=matnum, id=moodleid, name=info['fullname']))

        # Print for-loop progress
Christian Rohlfing's avatar
Christian Rohlfing committed
176
        if no_warn and not (cnt % max(1, round(num_students/10))):
177
            print(".", sep=' ', end='', flush=True)
178
179

    # Print results
Christian Rohlfing's avatar
Christian Rohlfing committed
180
    print("done.")
181
182
    print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
          .format(num_pdf=num_found_pdfs, num_csv=num_students))
Christian Rohlfing's avatar
Christian Rohlfing committed
183
    
184
185
186

    # Sanity check:
    # Check for PDFs not reflected in CSV (student not registered in Moodle)
187
    sanity_check(matnums_csv, matnums_folder)
188

189
    # Zip
190
191
192
    if not dry:
        # Zip
        print("Zipping")
193
        shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmp_folder)
194
        print('Zip archive is stored at {}'.format(outzip))
195
196

        # Delete temporary folder
197
        shutil.rmtree(tmp_folder)
198

199
    # Print dry run results
200
    else:
Christian Rohlfing's avatar
Christian Rohlfing committed
201
202
        dryout.sort()
        print("\nDry run results:\n{}".format("\n".join(dryout)))
203

204
    # Print status
205
206
207
    endtime = time.time()
    print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))
208
209


210
# Main routine
211
if __name__ == '__main__':
212
    main(sys.argv[1:])