Aufgrund einer Wartung wird GitLab am 19.10. zwischen 8:00 und 9:00 Uhr kurzzeitig nicht zur Verfügung stehen. / Due to maintenance, GitLab will be temporarily unavailable on 19.10. between 8:00 and 9:00 am.

Commit 6a45642b authored by Amrita Deb's avatar Amrita Deb
Browse files

Merge branch 'rohlfing-docstring-argument-handling' into 'master'

Automated documentation build

See merge request !39
parents 1661db24 9f907fd7
Pipeline #457905 passed with stages
in 7 minutes and 25 seconds
......@@ -19,12 +19,23 @@ class MainTest(unittest.TestCase):
def test_batch(self):
import batch
expected_files = ['moodle_feedbacks.zip', 'passwords.csv']
expected_folders = ['Four, Student_107_assignsubmission_file_']
expected_pdfs = ['123456_Nachname_w_aes.pdf']
expected_files = [
'moodle_feedbacks.zip',
'passwords.csv']
expected_folders = [
'LastnameA, FirstnameA_1519321_assignsubmission_file_',
'LastnameB, FirstnameB_1519322_assignsubmission_file_',
'LastnameJ, FirstnameJ_1519330_assignsubmission_file_',
'LastnameK, FirstnameK_1519331_assignsubmission_file_'
]
expected_pdfs = [
'123001_LastnameA_w_aes.pdf']
# Prepare parameter
in_dir = './pdfs'
in_dir = './tests/assets/pdfs'
csv = './tests/assets/Grades.csv'
dpi = 50
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
......@@ -35,8 +46,9 @@ class MainTest(unittest.TestCase):
os.mkdir(zipout_dir)
# Copy supplements file
batch.main(["-i", in_dir, "-o", out_dir, "-t", tmp_dir,
"--cores", "1", "--dpi", "100"])
batch.main([
in_dir, csv, out_dir,
"-t", tmp_dir, "--cores", "1", "--dpi", str(dpi)])
# Assert output
created_files = os.listdir(out_dir)
......
......@@ -18,22 +18,45 @@ class MainTest(unittest.TestCase):
# Clean up
shutil.rmtree(self.test_dir)
def test_encrypt_scan_single(self):
import encrypt
expected_files = [
'123001_LastnameA_aes.pdf']
# Prepare parameter
in_pdf = './tests/assets/pdfs/123001_LastnameA.pdf'
enc_pdf = '123001_LastnameA_aes.pdf'
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
# Encrypt files
encrypt.encrypt(
pdf_file=in_pdf, enc_file=os.path.join(out_dir, enc_pdf),
password='tests_are_fun')
created_files = os.listdir(out_dir)
created_files.sort()
self.assertEqual(expected_files, created_files)
def test_encrypt_scans(self):
import encrypt
expected_files = ['123456_Nachname_aes.pdf',
'456789_Lastname_aes.pdf',
'567891_Student_aes.pdf',
expected_files = ["123001_LastnameA_aes.pdf",
"123002_LastnameB_aes.pdf",
"123010_LastnameJ_aes.pdf",
"123011_LastnameK_aes.pdf",
'passwords.csv']
# Prepare parameter
in_dir = './pdfs'
in_dir = './tests/assets/pdfs'
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
# Encrypt files
encrypt.main(["-i", in_dir, "-o", out_dir])
encrypt.main([in_dir, out_dir])
created_files = os.listdir(out_dir)
created_files.sort()
......@@ -46,13 +69,13 @@ class MainTest(unittest.TestCase):
'passwords.csv']
# Prepare parameter
in_dir = './supplements'
in_dir = './tests/assets/supplements'
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
# Encrypt files
encrypt.main(["-i", in_dir, "-o", out_dir])
encrypt.main([in_dir, out_dir])
created_files = os.listdir(out_dir)
created_files.sort()
......
......@@ -14,15 +14,15 @@ class MainTest(unittest.TestCase):
def test_extract_grading_info(self):
from utils import moodle as moodle_utils
csv_file = "./Bewertungen.csv"
csv_file = "./tests/assets/Grades.csv"
gis = moodle_utils.extract_info(sheet_csv=csv_file)
self.assertEqual(gis[2]['lastname'], "Three")
self.assertEqual(gis[2]['lastname'], "LastnameJ")
num_students = moodle_utils.get_student_number(sheet_csv=csv_file)
self.assertEqual(num_students, 3)
self.assertEqual(num_students, 5)
csv_file = "./Grades.csv"
csv_file = "./tests/assets/Grades.csv"
gis = moodle_utils.extract_info(sheet_csv=csv_file)
# Check that "d'Lastname" gets whitened to "dLastname"
self.assertEqual(gis[3]['lastname'], "dLastname")
self.assertEqual(gis[0]['lastname'], "dLastname")
......@@ -18,15 +18,16 @@ class MainTest(unittest.TestCase):
# Clean up
shutil.rmtree(self.test_dir)
def test_prepare_moodle(self):
def test_preparemoodle_single(self):
import preparemoodle
expected_feedback_folder = 'Four, Student_107_assignsubmission_file_'
expected_feedback_file = '123456_Nachname.pdf'
expected_feedback_folder =\
'LastnameB, FirstnameB_1519322_assignsubmission_file_'
expected_feedback_file = '123002_LastnameB.pdf'
# Prepare parameter
in_dir = './pdfs'
sheet_csv = "./Bewertungen.csv"
in_dir = './tests/assets/pdfs'
sheet_csv = "./tests/assets/Grades.csv"
feedback_zip = 'feedbacks.zip'
tmp_dir = os.path.join(self.test_dir, 'tmp')
......@@ -35,8 +36,8 @@ class MainTest(unittest.TestCase):
out_zip = os.path.join(self.test_dir, feedback_zip)
# Call function
preparemoodle.main(["-i", in_dir, "--csv", sheet_csv,
"-t", tmp_dir, "-o", out_zip])
preparemoodle.main([in_dir, sheet_csv, out_zip,
"-t", tmp_dir])
# Unpack feedbacks
shutil.unpack_archive(out_zip, tmp_dir)
......
......@@ -19,13 +19,14 @@ class MainTest(unittest.TestCase):
import preparepdf
expected_files = [
'123456_F.pdf',
'123457_O.pdf',
'125412_T.pdf']
'123001_L.pdf',
'123002_L.pdf',
'123010_L.pdf',
'123011_L.pdf']
# Prepare parameter
in_zip = './submissions.zip'
sheet_csv = "./Bewertungen.csv"
in_zip = './tests/assets/submissions.zip'
sheet_csv = "./tests/assets/Grades.csv"
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
......@@ -35,8 +36,7 @@ class MainTest(unittest.TestCase):
# Call function
preparepdf.main([
"-i", in_zip, "-o", out_dir, "-c", sheet_csv,
"-t", tmp_dir])
in_zip, sheet_csv, out_dir, "-t", tmp_dir])
# Assert output
created_files = os.listdir(out_dir)
......
......@@ -16,7 +16,7 @@ class MainTest(unittest.TestCase):
expected_qr = "23-16"
pdf_file = "./pdfs/456789_Lastname.pdf"
pdf_file = "./tests/assets/pdfs/123001_LastnameA.pdf"
qr = qr_utils.first_qr_from_first_pdf_page(pdf_file=pdf_file)
self.assertEqual(qr, expected_qr)
......@@ -44,7 +44,7 @@ class MainTest(unittest.TestCase):
# Parameters
dpi = 200
pdf_file = "./pdfs/456789_Lastname.pdf"
pdf_file = "./tests/assets/pdfs/123001_LastnameA.pdf"
# Decode all QRs
qrs, _ = qr_utils.qrs_from_pdf(pdf_file=pdf_file, dpi=dpi)
......
......@@ -19,8 +19,8 @@ class MainTest(unittest.TestCase):
import renamescans
# Prepare parameter
in_dir = './pdfs_scan'
sheet_csv = "./Bewertungen.csv"
in_dir = './tests/assets/pdfs_scan'
sheet_csv = "./tests/assets/Grades.csv"
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
......@@ -28,7 +28,7 @@ class MainTest(unittest.TestCase):
# Call function
try:
renamescans.main([
"-i", in_dir, "-o", out_dir, "-c", sheet_csv,
in_dir, sheet_csv, out_dir,
"--dry", "--checkqr"])
except Exception:
pass
......@@ -37,19 +37,21 @@ class MainTest(unittest.TestCase):
import renamescans
expected_files = [
'123456_F.pdf',
'123457_O.pdf',
'125412_T.pdf']
'123001_L.pdf',
'123002_L.pdf',
'123010_L.pdf',
'123011_L.pdf',
'123012_d.pdf']
# Prepare parameter
in_dir = './pdfs_scan'
sheet_csv = "./Bewertungen.csv"
in_dir = './tests/assets/pdfs_scan'
sheet_csv = "./tests/assets/Grades.csv"
out_dir = os.path.join(self.test_dir, 'out')
os.mkdir(out_dir)
# Call function
renamescans.main(["-i", in_dir, "-o", out_dir, "-c", sheet_csv])
renamescans.main([in_dir, sheet_csv, out_dir])
# Assert output
created_files = os.listdir(out_dir)
......
......@@ -18,16 +18,19 @@ class MainTest(unittest.TestCase):
def test_supplements_from_pdf_folder(self):
import supplements
expected_files = ['123456_Nachname_GDET3_20H.pdf',
'123456_Nachname_GDET3_20H_loes.pdf',
'456789_Lastname_GDET3_20H.pdf',
'456789_Lastname_GDET3_20H_loes.pdf',
'567891_Student_GDET3_20H.pdf',
'567891_Student_GDET3_20H_loes.pdf']
expected_files = [
'123001_LastnameA_GDET3_20H.pdf',
'123001_LastnameA_GDET3_20H_loes.pdf',
'123002_LastnameB_GDET3_20H.pdf',
'123002_LastnameB_GDET3_20H_loes.pdf',
'123010_LastnameJ_GDET3_20H.pdf',
'123010_LastnameJ_GDET3_20H_loes.pdf',
'123011_LastnameK_GDET3_20H.pdf',
'123011_LastnameK_GDET3_20H_loes.pdf']
# Prepare parameter
supp_dir = './supplements'
pdf_dir = './pdfs'
supp_dir = './tests/assets/supplements'
pdf_dir = './tests/assets/pdfs'
supp_out_dir = os.path.join(self.test_dir, 'supplements_out')
os.mkdir(supp_out_dir)
......@@ -36,7 +39,7 @@ class MainTest(unittest.TestCase):
os.mkdir(tmp_dir)
# Copy supplements file
supplements.main(["-i", supp_dir, "-p", pdf_dir, "-o", supp_out_dir])
supplements.main([supp_dir, pdf_dir, supp_out_dir])
# Assert output
created_files = os.listdir(supp_out_dir)
......@@ -46,16 +49,21 @@ class MainTest(unittest.TestCase):
def test_supplements_from_csv(self):
import supplements
expected_files = ['123456_F_GDET3_20H.pdf',
'123456_F_GDET3_20H_loes.pdf',
'123457_O_GDET3_20H.pdf',
'123457_O_GDET3_20H_loes.pdf',
'125412_T_GDET3_20H.pdf',
'125412_T_GDET3_20H_loes.pdf']
expected_files = [
'123001_L_GDET3_20H.pdf',
'123001_L_GDET3_20H_loes.pdf',
'123002_L_GDET3_20H.pdf',
'123002_L_GDET3_20H_loes.pdf',
'123010_L_GDET3_20H.pdf',
'123010_L_GDET3_20H_loes.pdf',
'123011_L_GDET3_20H.pdf',
'123011_L_GDET3_20H_loes.pdf',
'123012_d_GDET3_20H.pdf',
'123012_d_GDET3_20H_loes.pdf']
# Prepare parameter
supp_dir = './supplements'
csv = 'Bewertungen.csv'
supp_dir = './tests/assets/supplements'
csv = './tests/assets/Grades.csv'
supp_out_dir = os.path.join(self.test_dir, 'supplements_out')
os.mkdir(supp_out_dir)
......@@ -64,7 +72,7 @@ class MainTest(unittest.TestCase):
os.mkdir(tmp_dir)
# Copy supplements file
supplements.main(["-i", supp_dir, "-p", csv, "-o", supp_out_dir])
supplements.main([supp_dir, csv, supp_out_dir])
# Assert output
created_files = os.listdir(supp_out_dir)
......@@ -75,16 +83,19 @@ class MainTest(unittest.TestCase):
import supplements
import watermark
expected_files = ['123456_Nachname_GDET3_20H_loes_w.pdf',
'123456_Nachname_GDET3_20H_w.pdf',
'456789_Lastname_GDET3_20H_loes_w.pdf',
'456789_Lastname_GDET3_20H_w.pdf',
'567891_Student_GDET3_20H_loes_w.pdf',
'567891_Student_GDET3_20H_w.pdf']
expected_files = [
'123001_LastnameA_GDET3_20H_loes_w.pdf',
'123001_LastnameA_GDET3_20H_w.pdf',
'123002_LastnameB_GDET3_20H_loes_w.pdf',
'123002_LastnameB_GDET3_20H_w.pdf',
'123010_LastnameJ_GDET3_20H_loes_w.pdf',
'123010_LastnameJ_GDET3_20H_w.pdf',
'123011_LastnameK_GDET3_20H_loes_w.pdf',
'123011_LastnameK_GDET3_20H_w.pdf']
# Prepare parameter
supp_dir = './supplements'
pdf_dir = './pdfs'
supp_dir = './tests/assets/supplements'
pdf_dir = './tests/assets/pdfs'
dpi = 100
supp_out_dir = os.path.join(self.test_dir, 'supplements_out')
......@@ -97,11 +108,11 @@ class MainTest(unittest.TestCase):
os.mkdir(out_dir)
# Copy supplements file
supplements.main(["-i", supp_dir, "-p", pdf_dir, "-o", supp_out_dir])
supplements.main([supp_dir, pdf_dir, supp_out_dir])
# Watermark files
watermark.main(["-i", supp_out_dir, "-o", out_dir,
"-t", tmp_dir, "--dpi", str(dpi)])
watermark.main(
[supp_out_dir, out_dir, "-t", tmp_dir, "--dpi", str(dpi)])
# Assert output
created_files = os.listdir(out_dir)
......
......@@ -22,11 +22,11 @@ class MainTest(unittest.TestCase):
import watermark
# Prepare parameter
in_dir = './pdfs'
in_dir = './tests/assets/pdfs'
dpi = 150
quality = 75
fontsize = 75
pdf_file = '123456_Nachname.pdf'
pdf_file = '123001_LastnameA.pdf'
tmp_dir = os.path.join(self.test_dir, 'tmp')
os.mkdir(tmp_dir)
......@@ -39,17 +39,18 @@ class MainTest(unittest.TestCase):
output_dir=out_dir, fontsize=fontsize, dpi=dpi,
quality=quality, pdf_file=pdf_file)
self.assertTrue(os.listdir(out_dir)[0], '123456_Nachname_w.pdf')
self.assertTrue(os.listdir(out_dir)[0], '123001_LastnameA_w.pdf')
def test_watermark_pdfs(self):
import watermark
expected_files = ['123456_Nachname_w.pdf',
'456789_Lastname_w.pdf',
'567891_Student_w.pdf']
expected_files = ["123001_LastnameA_w.pdf",
"123002_LastnameB_w.pdf",
"123010_LastnameJ_w.pdf",
"123011_LastnameK_w.pdf"]
# Prepare parameter
in_dir = './pdfs'
in_dir = './tests/assets/pdfs'
dpi = 150
tmp_dir = os.path.join(self.test_dir, 'tmp')
......@@ -59,9 +60,10 @@ class MainTest(unittest.TestCase):
os.mkdir(out_dir)
# Watermark files
watermark.main(["-i", in_dir, "-o", out_dir,
watermark.main([in_dir, out_dir,
"-t", tmp_dir, "--dpi", str(dpi)])
created_files = os.listdir(out_dir)
created_files.sort()
self.assertEqual(expected_files, created_files)
"""Common functions for handling Moodle grading sheet CSV
"""
import csv # handles csv
import argparse # argument parser for CSV files
def get_student_number(sheet_csv, csv_enc='utf-8'):
......@@ -46,8 +51,7 @@ def extract_info(sheet_csv, csv_delim=',', csv_quote='"', csv_enc='utf-8'):
csv_delim (str, optional): CSV delimiter. Defaults to ','.
csv_quote (str, optional): CSV quote char. Defaults to '"'.
csv_enc (str, optional): CSV encoding. Defaults to 'utf-8'.
Typical values: "'utf-8', 'utf-8-sig',
or 'cp1252' (Windows). "
Typical values: "'utf-8', 'utf-8-sig', or 'cp1252' (Windows). "
Returns:
list of dicts: grading information with following info per student:
......@@ -98,3 +102,16 @@ def extract_info(sheet_csv, csv_delim=',', csv_quote='"', csv_enc='utf-8'):
gi['fullname'] = fullname
return grading_infos
def get_moodle_csv_parser():
csv_parser = argparse.ArgumentParser(add_help=False)
csv_parser.add_argument(
"--csvdelim", default=",", help="CSV delimiter character.")
csv_parser.add_argument(
"--csvquote", default='"', help="CSV quote character.")
csv_parser.add_argument(
"--csvenc", default="utf-8", help="CSV encoding scheme. " +
"Typical encodings:'utf-8', 'utf-8-sig', or 'cp1252' (Windows).")
return csv_parser
......@@ -2,12 +2,17 @@
"""Watermarks each page of PDF with matriculation number
This scripts adds the matriculation number of students as watermark to their
respective exam scan PDF files
PDFs of exam scans from input folder are watermarked with the matriculation
number of the respective student. Watermarked PDFs are stored in output folder
Author: Amrita Deb <Deb@itc.rwth-aachen.de>
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de), " +\
"Rafael Goldbeck (rafael.goldbeck@isea.rwth-aachen.de)"
import sys # get arguments from command line
import os # path listing/manipulation/...
import time # keep track of time
......@@ -92,7 +97,8 @@ def create_watermark_template(img_file, matnum, fontsize, dpi):
template = Image.new('RGBA', newsize, (255, 255, 255, 0))
# Font
fnt = ImageFont.truetype('./fonts/arial.ttf', round(fontsize * dpi/250))
fnt = ImageFont.truetype(
'./assets/fonts/arial.ttf', round(fontsize*dpi/250))
# Drawing context
d = ImageDraw.Draw(template)
......@@ -111,9 +117,11 @@ def create_watermark_template(img_file, matnum, fontsize, dpi):
def remove_transparency(im, bg_colour=(255, 255, 255)):
"""
Correct transparent image turning black issue
Args:
im (PIL.Image.Image): pdf page image
bg_colour (tuple): background color white code
Returns:
PIL.Image.Image: corrected image when the image is transparent
else just return the pdf page image
......@@ -195,7 +203,7 @@ def combine_all_pdfs(pdf_pages, out_dir):
def watermark_pdf(input_dir, tmp_dir, output_dir,
fontsize, dpi, quality, pdf_file):
"""Watermarkes each page of a given PDF file
"""Watermarks each page of a given PDF file
Args:
input_dir (str): path to input directory
......@@ -212,7 +220,7 @@ def watermark_pdf(input_dir, tmp_dir, output_dir,
# img_files = convert_pdf_to_img(pdf_file, input_dir, tmp_dir, dpi)
img_files = convert_pdf_to_img(pdf_file, input_dir, tmp_dir, dpi)
# Extracting matriculation numebers
# Extracting matriculation numbers
matnum = matnum_utils.get_matnum(pdf_file)
# Watermarking PDF page images
......@@ -235,8 +243,40 @@ def watermark_pdf(input_dir, tmp_dir, output_dir,
return watermarked_pdf
def _make_parser():
parser = argparse.ArgumentParser(
description=__doc__, prog='watermark.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"infolder", help="Input folder with PDFs to be watermarked")
parser.add_argument(
"outfolder", help="Output folder with watermarked PDFs")
parser.add_argument(
"-f", "--fontsize", default="75",
help="Font size of watermark text in points")
parser.add_argument(
"-c", "--cores", default="1",
help="Number of cores for parallel processing")
parser.add_argument(
"-t", "--tmp", default="./tmp", help="Temporary folder")
parser.add_argument(
"-d", "--dpi", default="150",
help="Dots-per-inch parameter for PDF to image conversion")
parser.add_argument(
"-q", "--quality", default="75",
help="quality parameter for JPEG compression of watermarked PDF page")
return parser
# Create argument parser with default values
_parser = _make_parser()
def main(args):
"""Main function
"""Main routine
For all PDFs in ./pdfs folder:
1) Convert each page of the PDFs into image
......@@ -246,35 +286,11 @@ def main(args):
"""
# Argument handling
parser = argparse.ArgumentParser(description='''
PDFs of exam scans from folder 'in' are watermarked with the
matriculation number of the respective student.
Watermarked PDFs are stored in folder 'out'
''')
parser.add_argument("-i", "--infolder", default="./pdfs",
help="Input folder with PDFs. Default: ./pdfs")
parser.add_argument("-o", "--outfolder", default="./pdfs_watermarked",
help="Output folder of the PDFs. " +
"Default: ./pdfs_watermarked")
parser.add_argument("-f", "--fontsize", default="75",
help="Font size of watermark text in points. " +
"Default: 75")
parser.add_argument("-c", "--cores", default="1",
help="Number of cores for parallel processing. " +
"Default: 1")
parser.add_argument("-t", "--tmp", default="./tmp",
help="tmp folder. Default: ./tmp/")
parser.add_argument("-d", "--dpi", default="150",
help="DPI parameter for PDF to image conversion. " +
"Default: 150")
parser.add_argument("-q", "--quality", default="75",
help="quality parameter for jpeg. Default: 75")
args = parser.parse_args(args)
args = _parser.parse_args(args)
infolder = args.infolder
outfolder = args.outfolder
cores = int(args.cores)
tmpdir = args.tmp
cores = int(args.cores)
fontsize = int(args.fontsize)
dpi = int(args.dpi)
quality = int(args.quality)
......@@ -282,15 +298,31 @@ def main(args):
# Print status
starttime = time.time()
pdf_folder = os.listdir(infolder)
pdf_files = [_ for _ in pdf_folder
if _.lower().endswith(".pdf") and matnum_utils.starts_with_matnum(_)]
print("""
pdf_files = [
_ for _ in pdf_folder
if _.lower().endswith(".pdf") and matnum_utils.starts_with_matnum(_)]
if len(pdf_files) > 0:
print("""
Available PDFs to be watermarked:
- {}
Files in output folder {} will be overwritten during this process.
Parallel execution with {:d} cores from now on.
""".format("\n- ".join(pdf_files), outfolder, cores))
else:
print("""
There are no PDFs in the given directory.
Exiting now.""")
return