Skip to content

GitLab

Commit 79bd4629 authored by Christian Rohlfing's avatar Christian Rohlfing
Browse files

Beautification in argument handling and docstrings

parent 0048494f
Pipeline #456876 failed with stage
in 1 minute and 49 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 335 additions and 185 deletions
batch.py
View file @ 79bd4629
#!/usr/bin/env python
"""watermarks, encrypts and prepares PDF files for upload to Moodle assignment
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import os
import sys
import time
import argparse
import utils.moodle as moodle
import preparepdf
import supplements
import watermark
......@@ -10,42 +23,54 @@ import encrypt
import preparemoodle
def main(args):
# Argument handling
parser = argparse.ArgumentParser(description='''
Watermark and encrypts exams and prepares everything for moodle upload.
Attention: contents of folder 'out' will be deleted in the beginning!
''')
def _make_parser():
csv_parser = moodle.get_moodle_csv_parser()
parser = argparse.ArgumentParser(
parents=[csv_parser],
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"infolder", help="Input folder with PDFs.")
"infolder", help="Input folder with PDFs")
parser.add_argument(
"csv",
help="Moodle grading CSV file, needed to construct " +
"folder names for moodle zip")
"csv", help="Moodle grading CSV sheet")
parser.add_argument(
"outfolder",
help="output folder with passwords.csv and moodle_feedbacks.zip.")
help="output folder with passwords.csv and Moodle feedbacks ZIP")
parser.add_argument(
"-e", "--cores", default="2",
help="Number of cores for parallel processing. Default: 2")
"--supp", action='store_true',
help="Flag for watermarking supplements")
parser.add_argument(
"-p", "--password", default="",
help="sets global password. Default: empty, such that each PDF gets " +
"custom password")
"--suppinfolder", default="./supplements",
help="Input folder with supplements such as sample solutions")
parser.add_argument(
"-d", "--dpi", default="150",
help="DPI parameter for pdf to image conversion. Default: 150")
"--zip", default="0", help="ZIP file with submissions to be " +
"extracted to input folder")
parser.add_argument(
"-t", "--tmp", default="./tmp", help="Tmp folder. Default: ./tmp/")
"-e", "--cores", default="1",
help="Number of cores for parallel processing")
parser.add_argument(
"--suppinfolder", default="./supplements",
help="Input folder with sample solutions. Default: ./supplements")
"-p", "--password", default="",
help="sets global password. Default='' such that each PDF gets " +
"custom password")
parser.add_argument(
"--supp", action='store_true',
help="Flag for watermarking supplements.")
"-d", "--dpi", default="150",
help="Dots-per-inch parameter for PDF to image conversion")
parser.add_argument(
"--zip", default="0", help="Input zip file. Default: 0")
"-t", "--tmp", default="./tmp", help="Temporary folder")
return parser
# Create argument parser with default values
parser = _make_parser()
__doc__ += parser.format_help()
def main(args):
# Argument handling
args = parser.parse_args(args)
infolder = args.infolder
csv = args.csv
......@@ -57,6 +82,9 @@ def main(args):
watermark_supp = args.supp
supinfolder = args.suppinfolder
inzip = args.zip
csv_enc = args.csvenc
csv_delim = args.csvdelim
csv_quote = args.csvquote
starttime = time.time()
......@@ -64,19 +92,31 @@ def main(args):
if inzip != "0":
if not os.path.exists(infolder):
os.makedirs(infolder)
preparepdf.main([inzip, csv, infolder])
preparepdf.main([
inzip, csv, infolder,
'--csvenc', csv_enc, '--csvdelim', csv_delim,
'--csvquote', csv_quote])
# Watermarking
watermark_outfolder = os.path.join(tmp, 'pdfs_watermarked')
if not os.path.exists(watermark_outfolder):
os.makedirs(watermark_outfolder)
watermark.main([infolder, watermark_outfolder,
'--cores', cores, '--dpi', dpi])
watermark.main([
infolder, watermark_outfolder,
'--cores', cores, '--dpi', dpi])
if watermark_supp:
supoutfolder = os.path.join(tmp, 'supplements_out')
if not os.path.exists(supoutfolder):
os.makedirs(supoutfolder)
supplements.main([supinfolder, csv, supoutfolder])
supplements.main([
supinfolder, csv, supoutfolder,
'--csvenc', csv_enc, '--csvdelim', csv_delim,
'--csvquote', csv_quote
])
watermark.main([supoutfolder, watermark_outfolder,
'--cores', cores, '--dpi', dpi])
......@@ -94,7 +134,11 @@ def main(args):
moodle_tmp = os.path.join(tmp, 'tmp')
if not os.path.exists(moodle_tmp):
os.makedirs(moodle_tmp)
preparemoodle.main([enc_out, csv, moodle_out, '--tmp', moodle_tmp])
preparemoodle.main([
enc_out, csv, moodle_out,
'--tmp', moodle_tmp, '--csvenc', csv_enc, '--csvdelim', csv_delim,
'--csvquote', csv_quote])
endtime = time.time()
print(f'\nTotal time taken: {endtime-starttime:.2f}s\n')
......
encrypt.py
View file @ 79bd4629
......@@ -2,12 +2,16 @@
"""Creates encrypted copies of PDFs
This scripts creates encrypted copies of (watermarked) PDFs well as a CSV file
storing passwords for each file.
PDFs in input folder are encrypted and stored in output folder well as a CSV
file mapping passwords to each PDF.
Author: Amrita Deb <Deb@itc.rwth-aachen.de>
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import pikepdf
import os
import csv
......@@ -34,32 +38,42 @@ def encrypt(pdf_file, enc_file, password):
pdf.close()
def main(args):
"""Main function
1) Lists all PDFs to be encrypted from input folder
2) Encrypt pdf with randomly generated 8 character long password
3) Prepare a csv file that contains matriculation number and password
"""
def _make_parser():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Argument handling
parser = argparse.ArgumentParser(description='''
PDFs in input folder are encrypted and stored in output folder.
Alongside with a CSV file mapping passwords to each PDF.
''')
parser.add_argument(
"infolder", help="Input folder storing PDFs to be encrypted.")
"infolder", help="Input folder of PDFs to be encrypted.")
parser.add_argument(
"outfolder", help="Output folder storing encrypted PDFs.")
"outfolder", help="Output folder of encrypted PDFs.")
parser.add_argument(
"-p", "--password", default="",
help="Common password for all encrypted PDFs. " +
"Default: '' results in generation of random passwords.")
"Default='' results in generation of random passwords.")
parser.add_argument(
"-w", "--passwordout", default="",
"--passwordout", default="",
help="Output path for CSV file. " +
"Default: '' will be changed to [outfolder]/passwords.csv.")
"Default='' will be changed to [outfolder]/passwords.csv.")
return parser
# Create argument parser with default values
parser = _make_parser()
__doc__ += parser.format_help()
def main(args):
"""Main function
1) Lists all PDFs to be encrypted from input folder
2) Encrypt pdf with randomly generated 8 character long password
3) Prepare a csv file that contains matriculation number and password
"""
# Argument handling
args = parser.parse_args(args)
infolder = args.infolder
outfolder = args.outfolder
......
preparemoodle.py
View file @ 79bd4629
#!/usr/bin/env python
"""Prepares batch upload to Moodle's assignment module.
PDFs in folder 'in' are moved to a certain folder structure to be recognized
by moodle and finally zipped to 'outzip'.
Attention: Zip-archive 'outzip' will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import os
import time
import time # keep track of time
import shutil # copyfile, make_archive
import argparse # argument parsing
import sys
......@@ -132,42 +144,43 @@ def sanity_check(matnums_csv, matnums_folder):
return notfoundcsv, notfoundpdf
def main(args):
"""Main routine
"""
# Parse input arguments
parser = argparse.ArgumentParser(description='''
prepares batch upload to Moodle via assignment module.
PDFs in folder 'in' are moved to folder 'tmp' with a certain folder
structure and finally zipped to 'out'.
Attention: zip-archive 'out' will be overwritten in the following!
def _make_parser():
csv_parser = moodle.get_moodle_csv_parser()
parser = argparse.ArgumentParser(
parents=[csv_parser],
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
''')
parser.add_argument(
"infolder", help="Input folder with PDFs.")
parser.add_argument(
"csv", help="Moodle grading sheet.")
parser.add_argument(
"outzip", help="Zip archive with feedback files.")
parser.add_argument(
"--csvdelim", default=",", help="CSV delimiter. Default: ','")
parser.add_argument(
"--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
parser.add_argument(
"--csvenc", default="utf-8", help="CSV encoding scheme. " +
"Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
"Default: 'utf-8'")
parser.add_argument(
"-d", "--dry", action='store_true', help="Flag for dry run.")
parser.add_argument(
"-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
"-t", "--tmp", default="./tmp", help="Temporary folder.")
parser.add_argument(
"--nowarn", action='store_true', help="Disables warnings")
"--nowarn", action='store_true', help="Disables warnings.")
parser.add_argument(
"--moodlefilesize", default="250",
help="Moodle upload file size in MiB. Default: 250")
"--moodleuploadlimit", default="250",
help="Moodle upload limit in MiB.")
return parser
# Create argument parser with default values
parser = _make_parser()
__doc__ += parser.format_help()
def main(args):
"""Main routine
"""
# Parse input arguments
args = parser.parse_args(args)
infolder = args.infolder
sheet_csv = args.csv
......@@ -178,7 +191,7 @@ def main(args):
csv_delim = args.csvdelim
csv_quote = args.csvquote
csv_enc = args.csvenc
size_limit = int(args.moodlefilesize) # Moodle upload size limit in MiB
size_limit = int(args.moodleuploadlimit) # Moodle upload size limit in MiB
# Print status
starttime = time.time()
......@@ -223,9 +236,9 @@ Processing {} students'''.format(num_students))
matnums_csv = []
moodleids = []
if no_warn:
print("Start processing", sep=' ', end='', flush=True)
print("Start copying", sep=' ', end='', flush=True)
else:
print("Start processing")
print("Start copying")
for cnt, info in enumerate(infos):
# Copy PDF files
# Find all PDFs starting with matriculation number, e.g.
......@@ -300,8 +313,7 @@ Processing {} students'''.format(num_students))
# Print status
endtime = time.time()
print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))
print("Time taken: {:.2f}".format(endtime-starttime))
# Main routine
......
preparepdf.py
View file @ 79bd4629
#!/usr/bin/env python
"""Extract student's submission files from Moodle assignment
Transfer PDF files from ZIP file containing all submissions of a Moodle
assignment into output folder with file names following exam scan
naming convention.
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import sys # get arguments from command line
import os # path listing/manipulation/...
import time # keep track of time
......@@ -7,56 +22,58 @@ import shutil # unzipping and copying files
from utils import moodle as moodle
def main(args):
"""Transfer PDF files from submisions zip file (or already extracted folder)
containing all submissions into user provided folder following exam scan
naming convention
def _make_parser():
csv_parser = moodle.get_moodle_csv_parser()
1) Files are extracted from zip file location eg: ./all_submissions.zip
In case folder is given, extraction is skipped.
2) Scan through extracted folder for PDF files.
Only 1 PDF file/student is accepted.
3) Matriculation number and last name are fetched from grading worksheet
4) PDFs from extracted folder are renamed according to convention and
placed in user provided outfolder
"""
parser = argparse.ArgumentParser(
parents=[csv_parser],
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Argument handling
parser = argparse.ArgumentParser(description='''
Zip file 'inzip', containing all submissions of an assignment,
is extracted, renamed according to convention 'filenameformat'
and placed in folder 'outfolder'.
''')
parser.add_argument(
"inzip", help="Input zip file or already extracted folder.")
"inzip", help="Input ZIP file or extracted folder.")
parser.add_argument(
"csv", help="Moodle grading sheet.")
parser.add_argument(
"outfolder", help="Output folder with PDFs.")
parser.add_argument(
"--filenameformat", default="{matnum}_{fullname[0]}",
"-f", "--filenameformat", default="{matnum}_{fullname[0]}",
help="File name format. Available keywords: " +
"{matnum}, {fullname}, {lastname}, {firstname}. " +
"Default: '{matnum}_{fullname[0]}'")
parser.add_argument(
"--copyall", action='store_true',
"-c", "--copyall", action='store_true',
help="If set, copies all files (including multiple and non-PDF files)")
parser.add_argument(
"--appendoriginal", action='store_true',
"-a", "--appendoriginal", action='store_true',
help="If set, appends original file name to new location's file name")
parser.add_argument(
"--csvdelim", default=",", help="CSV delimiter. Default: ','")
parser.add_argument(
"--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
parser.add_argument(
"--csvenc", default="utf-8", help="CSV encoding scheme. " +
"Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
"Default: 'utf-8'")
parser.add_argument(
"-d", "--dry", action='store_true', help="Flag for dry run.")
parser.add_argument(
"-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
"-t", "--tmp", default="./tmp", help="Temporary folder.")
return parser
# Create argument parser with default values
parser = _make_parser()
__doc__ += parser.format_help()
def main(args):
"""Main routine
1) Files are extracted from zip file location eg: ./all_submissions.zip
In case folder is given, extraction is skipped.
2) Scan through extracted folder for PDF files.
Only 1 PDF file/student is accepted.
3) Matriculation number and last name are fetched from grading worksheet
4) PDFs from extracted folder are renamed according to convention and
placed in user provided outfolder
"""
# Argument handling
args = parser.parse_args(args)
inzip = args.inzip
outfolder = args.outfolder
......
renamescans.py
View file @ 79bd4629
#!/usr/bin/env python
"""Rename scanned PDFs assuming scan order equal to alphabetical order of
students in Moodle grading sheet.
Renames scans accordingly to info in Moodle grading sheet, such that the
file name starts with the matriculation number. This only works if exams were
scanned in alphabetical order.
Optionally, each scanned PDF is searched for barcodes/QRs containing the
matriculation number to double check.
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import os
import time
import shutil # copyfile, make_archive
......@@ -11,45 +27,44 @@ import utils.matnum as matnum_utils
import utils.qr as qr_utils
def main(args):
"""Main routine
"""
def _make_parser():
csv_parser = moodle.get_moodle_csv_parser()
parser = argparse.ArgumentParser(
parents=[csv_parser],
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Parse input arguments
parser = argparse.ArgumentParser(description='''
renames scans accordingly to info in Moodle grading sheet, such that
the file name starts with the matriculation number.
This only works if exams were scanned in alphabetical order.
Optionally, each scanned PDF is searched for barcodes/QRs containing
the matriculation number to double check.
Attention: Contents in output folder are overwritten!
''')
parser.add_argument(
"infolder", help="Input folder with PDFs.")
parser.add_argument(
"csv", help="Moodle grading sheet.")
parser.add_argument(
"outfolder", help="Output folder with renamed scans.")
parser.add_argument(
"--filenameformat", default="{matnum}_{fullname[0]}",
help="File name format. Available keywords: " +
"{matnum}, {fullname}, {lastname}, {firstname}. " +
"Default: '{matnum}_{fullname[0]}'")
parser.add_argument(
"--csvdelim", default=",", help="CSV delimiter. Default: ','")
parser.add_argument(
"--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
parser.add_argument(
"--csvenc", default="utf-8", help="CSV encoding scheme. " +
"Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
"Default: 'utf-8'")
parser.add_argument(
"-q", "--checkqr", action='store_true',
help="Flag for additional QR code match.")
parser.add_argument(
"-d", "--dry", action='store_true', help="Flag for dry run.")
return parser
# Create argument parser with default values
parser = _make_parser()
__doc__ += parser.format_help()
def main(args):
"""Main routine
"""
# Parse input arguments
args = parser.parse_args(args)
infolder = args.infolder
sheet_csv = args.csv
......@@ -148,8 +163,7 @@ def main(args):
# Print time
endtime = time.time()
print("""Done.
Time taken: {:.2f}""".format(endtime-starttime))
print("Time taken: {:.2f}".format(endtime-starttime))
# Main routine
......
supplements.py
View file @ 79bd4629
......@@ -3,11 +3,18 @@
"""Prepare supplement material
This script copies and renames supplementary material (such as exam sheet or
sample solution) to have the prefix ("[matnum]_[lastname]").
sample solution) with file names following the exam scan naming convention.
This information is either taken from the filenames of exam scan PDFs or from
the Moodle grading CSV file.
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import sys # get arguments from command line
import os # path listing/manipulation/...
import time # keep track of time
......@@ -71,43 +78,44 @@ def copy_supplements(supp_dir, supp_files, prefixes, output_dir, dry=False):
return copied_files
def main(args):
"""Main function
"""
def _make_parser():
csv_parser = moodle.get_moodle_csv_parser()
parser = argparse.ArgumentParser(
parents=[csv_parser],
description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# Argument handling
parser = argparse.ArgumentParser(description='''
This script copies supplementary material (such as exam sheet or sample
solution) to have the prefix (e.g. "[matnum]_[lastname]").
This information is either taken from the filenames of exam scan PDFs or
from the Moodle grading CSV file.
''')
parser.add_argument(
"infolder",
help="Folder with supplements.")
"infolder", help="Folder with supplements.")
parser.add_argument(
"prefix",
help="Provides information to construct prefixes. Either PDF folder " +
"with scanned PDFs or Moodle grading CSV file.")
"prefix", help="Provides information to construct prefixes. Either " +
"folder with scanned PDFs or Moodle grading CSV file.")
parser.add_argument(
"outfolder",
help="Output folder with supplements per student.")
parser.add_argument(
"--filenameformat", default="{matnum}_{fullname[0]}",
help="File name format. Available keywords: " +
"{{matnum}}, {{fullname}}, {{lastname}}, {{firstname}}. " +
"Default: '{{matnum}}_{{fullname[0]}}'")
parser.add_argument(
"--csvdelim", default=",", help="CSV delimiter. Default: ','")
parser.add_argument(
"--csvquote", default='"', help="CSV quote char." + """Default: '"'""")
parser.add_argument(
"--csvenc", default="utf-8", help="CSV encoding scheme. " +
"Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows). " +
"Default: 'utf-8'")
parser.add_argument(