preparemoodle.py 7.01 KB
Newer Older
1
2
#!/usr/bin/env python

3
4
import os
import time
5
import shutil  # copyfile, make_archive
6
import argparse  # argument parsing
7
import sys
8
9

import utils.moodle as moodle
10
import utils.matnum as matnum_utils
11
12


13
14
15
16
def sanity_check(matnums_csv, matnums_folder):
    """Check two cases for sanity:
    - Are there PDF files with no corresponding CSV entries?
    - Are there CSV entries with no provided PDF file?
17
18

    Args:
19
20
        matnums_csv (list): Matnums of all CSV entries
        matnums_folder (list): Matnums of all provided PDF files
21
22
    """

23
24
    # PDF files with no entry in CSV:
    notfoundcsv = list(set(matnums_folder).difference(matnums_csv))
25

26
27
    # Entries in CSV without PDF file
    notfoundpdf = list(set(matnums_csv).difference(matnums_folder))
28
29

    # Report back
30
31
32
33
34
35
36
    if len(notfoundcsv) > 0:
        print('''Warning: Following {} matnums have PDFs but no entry in CSV:
            {}'''.format(len(notfoundcsv), ", ".join(notfoundcsv)))

    if len(notfoundpdf) > 0:
        print('''Warning: Following {} matnums have CSV entries but no PDF:
            {}'''.format(len(notfoundpdf), ", ".join(notfoundpdf)))
37
38
39

    print("Done.\n")

40
41
    return notfoundcsv, notfoundpdf

42
43
44
45
46
47
48
49

def main(args):
    """Main routine
    """

    # Parse input arguments
    parser = argparse.ArgumentParser(description='''
    prepares batch upload to Moodle via assignment module.
50
51
    PDFs in folder 'in' are moved to folder 'tmp' with a certain folder
    structure and finally zipped to 'out'.
52
53
54
55
    Attention: zip-archive 'out' will be overwritten in the following!

    ''')
    parser.add_argument("-i", "--infolder", default="./pdfs_encrypted",
56
57
                        help="Input folder with PDFs." +
                             "Default: ./pdfs_encrypted")
58
    parser.add_argument("-c", "--csv", default="./Bewertungen.csv",
59
60
61
62
63
64
65
66
67
68
69
                        help="Moodle grading sheet, needed to construct " +
                             "submission folder names. " +
                             "Default: ./Bewertungen.csv")
    parser.add_argument("--csvdelim", default=",",
                        help="CSV delimiter. Default: ','")
    parser.add_argument("--csvquote", default='"', help="Quote character." +
                        """Default: '"'""")
    parser.add_argument("--csvencoding", default="utf-8",
                        help="CSV encoding scheme. Typical encodings:" +
                             "'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
                             "Default: 'utf-8'")
70
    parser.add_argument("-o", "--outzip", default="./moodle_feedbacks.zip",
71
                        help="Zip archive. Default: ./moodle_feedbacks.zip")
72
    parser.add_argument("-d", "--dry", action='store_true',
73
                        help="Flag for dry run, displays folder structure")
74
75
76
77
78
79
80
    parser.add_argument("-t", "--tmp", default="./tmp",
                        help="tmp folder. Default: ./tmp")
    parser.add_argument("--nowarn", action='store_true',
                        help="Disables warnings")

    args = parser.parse_args(args)
    infolder = args.infolder
81
    sheet_csv = args.csv
82
83
84
85
    outzip = args.outzip
    tmpfolder = os.path.join(args.tmp, "to_be_zipped_for_moodle")
    dry = args.dry
    nowarn = args.nowarn
86
87
88
    csvdelim = args.csvdelim
    csvquote = args.csvquote
    csvenc = args.csvencoding
89

90
    # Print number of lines
91
    starttime = time.time()
92
93
    numstudents = moodle.get_student_number(sheet_csv=sheet_csv,
                                            csv_enc=csvenc)
94
95
96

    print('''Preparing for moodle upload
Processing {} lines
97
  '''.format(numstudents))
98

99
    # Clean up and create temporary folder
100
101
102
103
104
105
106
107
108
109
110
111
    dryout = ""
    if dry:
        print("Dry run\n")
    else:
        # Remove zip file
        if os.path.exists(outzip):
            os.remove(outzip)

        # Create temporary folder within given temporary directory
        if not os.path.isdir(tmpfolder):
            os.mkdir(tmpfolder)

112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    # Parse input folder
    # Only PDF files are considered with first digits
    # containing matriculation number
    matnums_folder = []
    allfiles = os.listdir(infolder)
    allfiles.sort()
    allpdfs = []
    for f in allfiles:
        if f.lower().endswith('.pdf') and matnum_utils.starts_with_matnum(f):
            allpdfs.append(f)
            matnums_folder.append(matnum_utils.get_matnum(f))

    # Parse grading infos from CSV file
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csvdelim,
                                csv_quote=csvquote, csv_enc=csvenc)

    # Loop over grading infos
    numfoundpdfs = 0
    matnums_csv = []
    moodleids = []
    for cnt, info in enumerate(infos):
        # Copy PDF files
        # Find all PDFs starting with matriculation number, e.g.
        # '123456_Lastname_sheet.pdf' and '123456_Lastname_exam.pdf'
        # If pdf files for current student exists, create a directory and
        # copy the pdf files to it. The resulting directories can be
        # uploaded to Moodle
        matnum = info['matnum']
        matnums_csv.append(matnum)
        moodleid = info['moodleid']
        moodleids.append(moodleid)

        pdfs_student = [_ for _ in allpdfs
                        if matnum == matnum_utils.get_matnum(_)]
        if len(pdfs_student) > 0:  # Found at least one pdf
            numfoundpdfs += len(pdfs_student)

            # Prepare submission folder
            folder = moodle.submission_folder_name(info)
            longfolder = os.path.join(tmpfolder, folder)

            # Create folder
            if not dry:
                os.mkdir(longfolder)

            # Copy all files to folder
            for pdffile in pdfs_student:
                longpdffile = os.path.join(infolder, pdffile)
                longpdffiledest = os.path.join(longfolder, pdffile)
161
                if not dry:
162
163
164
165
166
167
168
169
170
171
172
                    shutil.copyfile(longpdffile, longpdffiledest)
                else:
                    dryout += "\n{}".format(os.path.join(folder, pdffile))

        elif not nowarn:  # No PDF found
            print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
                  .format(matnum=matnum, id=moodleid, name=info['fullname']))

        # Print for-loop progress
        if not (cnt % max(1, round(numstudents/10))):
            print(".", sep=' ', end='', flush=True)
173
174

    # Print results
175
176
    print("Found {numpdf} PDFs (CSV had {numcsv} entries)"
          .format(numpdf=numfoundpdfs, numcsv=numstudents))
177
178
179
180
    print("done.")

    # Sanity check:
    # Check for PDFs not reflected in CSV (student not registered in Moodle)
181
    sanity_check(matnums_csv, matnums_folder)
182

183
    # Zip
184
185
186
187
    if not dry:
        # Zip
        print("Zipping")
        shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmpfolder)
188
        print('Zip archive is stored at {}'.format(outzip))
189
190
191
192

        # Delete temporary folder
        shutil.rmtree(tmpfolder)

193
    # Print dry run results
194
195
196
    else:
        print("\nDry run results:\n{}".format(dryout))

197
    # Print status
198
199
200
    endtime = time.time()
    print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))
201
202


203
# Main routine
204
if __name__ == '__main__':
205
    main(sys.argv[1:])