preparemoodle.py 8.39 KB
Newer Older
1
2
#!/usr/bin/env python

3
4
import os
import time
5
import shutil  # copyfile, make_archive
6
import argparse  # argument parsing
7
import sys
8
9

import utils.moodle as moodle
10
import utils.matnum as matnum_utils
11

Deb's avatar
Deb committed
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import zipfile


def bytesto(bytes, to, bsize=1024):
    """convert bytes to megabytes, etc.
       sample code:
           print('mb= ' + str(bytesto(314575262000000, 'm')))
       sample output: 
           mb= 300002347.946
    """

    a = {'k' : 1, 'm': 2, 'g' : 3, 't' : 4, 'p' : 5, 'e' : 6 }
    r = float(bytes)
    for i in range(a[to]):
        r = r / bsize

    return(round(r,2))
29

30
31
32
33
def sanity_check(matnums_csv, matnums_folder):
    """Check two cases for sanity:
    - Are there PDF files with no corresponding CSV entries?
    - Are there CSV entries with no provided PDF file?
34
35

    Args:
36
37
        matnums_csv (list): Matnums of all CSV entries
        matnums_folder (list): Matnums of all provided PDF files
38
39
    """

40
41
    # PDF files with no entry in CSV:
    notfoundcsv = list(set(matnums_folder).difference(matnums_csv))
42

43
44
    # Entries in CSV without PDF file
    notfoundpdf = list(set(matnums_csv).difference(matnums_folder))
45
46

    # Report back
47
48
49
50
51
52
53
    if len(notfoundcsv) > 0:
        print('''Warning: Following {} matnums have PDFs but no entry in CSV:
            {}'''.format(len(notfoundcsv), ", ".join(notfoundcsv)))

    if len(notfoundpdf) > 0:
        print('''Warning: Following {} matnums have CSV entries but no PDF:
            {}'''.format(len(notfoundpdf), ", ".join(notfoundpdf)))
54
55
56

    print("Done.\n")

57
58
    return notfoundcsv, notfoundpdf

59
60
61
62
63
64
65
66

def main(args):
    """Main routine
    """

    # Parse input arguments
    parser = argparse.ArgumentParser(description='''
    prepares batch upload to Moodle via assignment module.
67
68
    PDFs in folder 'in' are moved to folder 'tmp' with a certain folder
    structure and finally zipped to 'out'.
69
70
71
    Attention: zip-archive 'out' will be overwritten in the following!

    ''')
Deb's avatar
Deb committed
72
73
    parser.add_argument("infolder", help="Input folder with PDFs.")
    parser.add_argument("csv", help="Moodle grading sheet.")
74
75
76
77
78
79
80
81
    parser.add_argument(
        "--csvdelim", default=",", help="CSV delimiter. Default: ','")
    parser.add_argument(
        "--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
    parser.add_argument(
        "--csvenc", default="utf-8", help="CSV encoding scheme. " +
        "Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
        "Default: 'utf-8'")
Deb's avatar
Deb committed
82
    parser.add_argument("outzip", help="Zip archive.")
83
84
85
86
    parser.add_argument(
        "-d", "--dry", action='store_true', help="Flag for dry run.")
    parser.add_argument(
        "-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
Deb's avatar
Deb committed
87
88
89
    parser.add_argument(
        "--moodlefilesize", default="250",
        help="Moodle upload file size in MiB. Default: 250")
90
91
    parser.add_argument(
        "--nowarn", action='store_true', help="Disables warnings")
92
93
94

    args = parser.parse_args(args)
    infolder = args.infolder
95
    sheet_csv = args.csv
96
    outzip = args.outzip
97
    tmp_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle")
98
    dry = args.dry
99
100
101
102
    no_warn = args.nowarn
    csv_delim = args.csvdelim
    csv_quote = args.csvquote
    csv_enc = args.csvenc
Deb's avatar
Deb committed
103
    size_limit = int(args.moodlefilesize)  # Moodle upload size limit in MiB
104

105
    # Print status
106
    starttime = time.time()
107
108
    num_students = moodle.get_student_number(sheet_csv=sheet_csv,
                                             csv_enc=csv_enc)
109
110

    print('''Preparing for moodle upload
Christian Rohlfing's avatar
Christian Rohlfing committed
111
Processing {} students'''.format(num_students))
112

113
    # Clean up and create temporary folder
Christian Rohlfing's avatar
Christian Rohlfing committed
114
    dryout = []
115
    if dry:
Christian Rohlfing's avatar
Christian Rohlfing committed
116
        print("Dry run")
117
118
119
120
121
122
    else:
        # Remove zip file
        if os.path.exists(outzip):
            os.remove(outzip)

        # Create temporary folder within given temporary directory
123
124
        if not os.path.isdir(tmp_folder):
            os.mkdir(tmp_folder)
125

126
127
128
129
130
131
132
    # Parse input folder
    # Only PDF files are considered with first digits
    # containing matriculation number
    matnums_folder = []
    allfiles = os.listdir(infolder)
    allfiles.sort()
    allpdfs = []
Deb's avatar
Deb committed
133
134
135
    if (len(allfiles) == 0):
        print(""" There are no PDFs in the given directory. Exiting now.""")
        return
136
137
138
139
140
141
    for f in allfiles:
        if f.lower().endswith('.pdf') and matnum_utils.starts_with_matnum(f):
            allpdfs.append(f)
            matnums_folder.append(matnum_utils.get_matnum(f))

    # Parse grading infos from CSV file
142
143
    infos = moodle.extract_info(sheet_csv=sheet_csv, csv_delim=csv_delim,
                                csv_quote=csv_quote, csv_enc=csv_enc)
144
145

    # Loop over grading infos
146
    num_found_pdfs = 0
147
148
    matnums_csv = []
    moodleids = []
Christian Rohlfing's avatar
Christian Rohlfing committed
149
150
151
152
    if no_warn:
        print("Start processing", sep=' ', end='', flush=True)
    else:
        print("Start processing")
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    for cnt, info in enumerate(infos):
        # Copy PDF files
        # Find all PDFs starting with matriculation number, e.g.
        # '123456_Lastname_sheet.pdf' and '123456_Lastname_exam.pdf'
        # If pdf files for current student exists, create a directory and
        # copy the pdf files to it. The resulting directories can be
        # uploaded to Moodle
        matnum = info['matnum']
        matnums_csv.append(matnum)
        moodleid = info['moodleid']
        moodleids.append(moodleid)

        pdfs_student = [_ for _ in allpdfs
                        if matnum == matnum_utils.get_matnum(_)]
        if len(pdfs_student) > 0:  # Found at least one pdf
168
            num_found_pdfs += len(pdfs_student)
169
170
171

            # Prepare submission folder
            folder = moodle.submission_folder_name(info)
172
            longfolder = os.path.join(tmp_folder, folder)
Deb's avatar
Deb committed
173
            
174
175
176
177
178
179
180
181
            # Create folder
            if not dry:
                os.mkdir(longfolder)

            # Copy all files to folder
            for pdffile in pdfs_student:
                longpdffile = os.path.join(infolder, pdffile)
                longpdffiledest = os.path.join(longfolder, pdffile)
182
                if not dry:
183
184
                    shutil.copyfile(longpdffile, longpdffiledest)
                else:
Christian Rohlfing's avatar
Christian Rohlfing committed
185
186
187
                    dryout.append(
                        "- {old} -> {new}"
                        .format(old=pdffile, new=os.path.join(folder, pdffile)))
188

189
        elif not no_warn:  # No PDF found
190
191
192
193
            print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
                  .format(matnum=matnum, id=moodleid, name=info['fullname']))

        # Print for-loop progress
Christian Rohlfing's avatar
Christian Rohlfing committed
194
        if no_warn and not (cnt % max(1, round(num_students/10))):
195
            print(".", sep=' ', end='', flush=True)
196
197

    # Print results
Christian Rohlfing's avatar
Christian Rohlfing committed
198
    print("done.")
199
200
    print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
          .format(num_pdf=num_found_pdfs, num_csv=num_students))
Christian Rohlfing's avatar
Christian Rohlfing committed
201
    
202
203
204

    # Sanity check:
    # Check for PDFs not reflected in CSV (student not registered in Moodle)
205
    sanity_check(matnums_csv, matnums_folder)
206

207
    # Zip
208
    if not dry:
Deb's avatar
Deb committed
209
210
211
212
213
214
215
216
217
        foldersize = 0
        count = 1
        z = zipfile.ZipFile(os.path.splitext(outzip)[0]+str(count)+".zip", "w")
        for dirpath, dirnames, filenames in os.walk(tmp_folder):
            for file in filenames:
                file_path = os.path.join(dirpath, file)
                
                if not os.path.islink(file_path):
                    foldersize += os.path.getsize(file_path)
Deb's avatar
Deb committed
218
                    if bytesto(foldersize,'m') < size_limit:
Deb's avatar
Deb committed
219
220
221
222
223
224
225
226
227
228
229
                        z.write(file_path,os.path.join(os.path.relpath(file_path, tmp_folder),file))
                        os.remove(file_path)
                    else:
                        print("Preparing zip file "+str(count))
                        z.close()
                        print('Zip archive is stored at {}'.format(os.path.splitext(outzip)[0]+str(count)+".zip"))
                        count+=1
                        foldersize = 0
                        z = zipfile.ZipFile(os.path.splitext(outzip)[0]+str(count)+".zip", "w")
                        z.write(file_path,os.path.join(os.path.relpath(file_path, tmp_folder),file))
                        os.remove(file_path)
230
231

        # Delete temporary folder
232
        shutil.rmtree(tmp_folder)
233

234
    # Print dry run results
235
    else:
Christian Rohlfing's avatar
Christian Rohlfing committed
236
237
        dryout.sort()
        print("\nDry run results:\n{}".format("\n".join(dryout)))
238

239
    # Print status
240
241
242
    endtime = time.time()
    print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))
243
244


245
# Main routine
246
if __name__ == '__main__':
247
    main(sys.argv[1:])