Commit 6a45642b authored by Amrita Deb's avatar Amrita Deb
Browse files

Merge branch 'rohlfing-docstring-argument-handling' into 'master'

Automated documentation build

See merge request !39
parents 1661db24 9f907fd7
Pipeline #457905 passed with stages
in 7 minutes and 25 seconds
......@@ -6,3 +6,4 @@ pdfs_encrypted/*
.vscode
**/__pycache__
venv/*
......@@ -4,6 +4,7 @@ services:
stages:
- build
- deploy
- test
variables:
......@@ -25,10 +26,26 @@ build:
before_script:
- docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
pages:
image:
name: ${CI_REGISTRY_IMAGE}:latest
entrypoint: [""]
stage: deploy
script:
- pip3 install -r docs/source/requirements.txt
- ~/.local/bin/sphinx-build -b html docs/source/ public
artifacts:
paths:
- public
only:
- master
test:
stage: test
script:
- docker run --name examscan --rm -v $(pwd):$(pwd) -w $(pwd) ${CI_REGISTRY_IMAGE}:latest batch.py --i pdfs --o out
- mkdir out
- chmod 777 out
- docker run --name examscan --rm -v $(pwd):$(pwd) -w $(pwd) ${CI_REGISTRY_IMAGE}:latest watermark.py tests/assets/pdfs out
artifacts:
paths:
- out/
......@@ -48,3 +65,16 @@ test-coverage:
- reports/coverage.xml
reports:
cobertura: reports/coverage.xml
test-docs:
image:
name: ${CI_REGISTRY_IMAGE}:latest
entrypoint: [""]
stage: test
script:
- pip3 install -r docs/source/requirements.txt
- ~/.local/bin/sphinx-build -b html docs/source/ public
only:
- branches
except:
- master
ID,"Vollständiger Name",Matrikelnummer,Status,Bewertung,"Status des Bewertungsworkflows","Bewertung kann geändert werden","Zuletzt geändert (Abgabe)","Zuletzt geändert (Bewertung)","Feedback als Kommentar"
Teilnehmer/in107,"Four, Student",123456,"Zur Bewertung abgegeben - Unbewertet",,Unbewertet,Ja,"Montag, 18. Mai 2020, 10:11",-,
Teilnehmer/in106,"One, Student",123457,"Keine Abgabe - Unbewertet",,Unbewertet,Ja,-,"Samstag, 16. Mai 2020, 12:15",-,
Teilnehmer/in105,"Three, Student",125412,"Keine Abgabe - Unbewertet",,Unbewertet,Ja,-,"Samstag, 16. Mai 2020, 12:15",-,
\ No newline at end of file
Identifier,"Full name","Registration number",Status,Grade,"Grade can be changed","Last modified (grade)","Feedback comments"
"Participant 1519324","Lastname, First",321121,,,Yes,"Monday, 15 March 2021, 5:42 PM",
"Participant 1519323","Yet another last name, First Name",321122,,,Yes,"Monday, 15 March 2021, 5:42 PM",
"Participant 1519322","Last, First",321123,,,Yes,"Monday, 15 March 2021, 5:42 PM",
"Participant 1519321","d'Lastname, Firstname",321124,,,Yes,"Monday, 15 March 2021, 5:42 PM",
......@@ -52,7 +52,7 @@ Exemplary outputs can be downloaded:
python ./watermark.sh --help
```
The current version of code was tested on Windows10, Ubuntu 20.04.1 LTS and macOS 10.14 Mojave to ensure platform independance.The code has the following software dependencies which needs to installed before the programs can be run successfully. Specific version numbers along with categorisation based on script requirement is available [here](Dependancies.md):
The current version of code was tested on Windows10, Ubuntu 20.04.1 LTS and macOS 10.14 Mojave to ensure platform independence.The code has the following software dependencies which needs to installed before the programs can be run successfully. Specific version numbers along with categorisation based on script requirement is available [here](Dependancies.md):
* Imagemagick
* Ghostscript
* Python 3.8/3.9
......@@ -91,7 +91,7 @@ docker run --name examscan --rm -v $(pwd):$(pwd) -w $(pwd) examscan:latest batch
Run `preparepdf.py` (if you have submissions in a zip and not as scans), `supplements.py` (if you want to add watermarked sample solutions as well), `watermark.py`, `encrypt.py`, and `preparemoodle.py` (or run `batch.py` which runs all) as described in the sections below. In summary, these steps will
1. unzip all PDF files from the zip and rename them according to the schema `<Matriculationnumber>_<Lastname>`
1. unzip all PDF files from the zip and rename them according to the schema `<Matriculation number>_<Lastname>`
1. prepare sample solution for each students
1. watermark each page of each PDF with the corresponding matriculation number,
1. encrypt each PDF with a password (global or per-student) and
......@@ -135,6 +135,8 @@ python watermark.py --in ./pdfs --out ./pdfs_watermarked --cores 2
Folder `pdfs_watermarked` contains watermarked PDFs, with each page watermarked with the matriculation number of the student.
**TIP:** Play around with `dpi` and `quality` parameters according to your requirements. Higher values for these two will result in high resolution PDFs of bigger size (ideal for when the number of files is low). Lower values will result in PDFs having lower file size and low resolution (ideal when the number of files is high)
#### Watermark Sample solution copies
We assume that the folder `./supplements_out` holds the copies for every students
......@@ -145,7 +147,7 @@ python watermark.py --in ./supplements_out --out ./pdfs_watermarked --cores 2
#### Encrypt the files
Use either a global password by specifying it with the `--password` option or per-student passwords by ommiting `--password`.
Use either a global password by specifying it with the `--password` option or per-student passwords by omitting `--password`.
```bash
python encrypt.py --in ./pdfs_watermarked --out ./pdfs_encrypted --password ganzgeheim
......
#!/usr/bin/env python
"""watermarks, encrypts and prepares PDF files for upload to Moodle assignment
Attention: Contents in output folder will be overwritten in the following!
"""
__author__ = "Amrita Deb (deb@itc.rwth-aachen.de), " +\
"Christian Rohlfing (rohlfing@ient.rwth-aachen.de)"
import os
import sys
import time
import argparse
import utils.moodle as moodle
import preparepdf
import supplements
import watermark
......@@ -10,54 +23,54 @@ import encrypt
import preparemoodle
def _make_parser():
csv_parser = moodle.get_moodle_csv_parser()
parser = argparse.ArgumentParser(
parents=[csv_parser],
description=__doc__, prog='batch.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"infolder", help="Input folder with PDFs")
parser.add_argument(
"csv", help="Moodle grading CSV sheet")
parser.add_argument(
"outfolder",
help="output folder with passwords.csv and Moodle feedbacks ZIP")
parser.add_argument(
"--supp", action='store_true',
help="Flag for watermarking supplements")
parser.add_argument(
"--suppinfolder", default="./supplements",
help="Input folder with supplements such as sample solutions")
parser.add_argument(
"--zip", default="0", help="ZIP file with submissions to be " +
"extracted to input folder")
parser.add_argument(
"-e", "--cores", default="1",
help="Number of cores for parallel processing")
parser.add_argument(
"-p", "--password", default="",
help="sets global password. Default='' such that each PDF gets " +
"custom password")
parser.add_argument(
"-d", "--dpi", default="150",
help="Dots-per-inch parameter for PDF to image conversion")
parser.add_argument(
"-t", "--tmp", default="./tmp", help="Temporary folder")
return parser
# Create argument parser with default values
_parser = _make_parser()
def main(args):
# Argument handling
parser = argparse.ArgumentParser(description='''
Watermark and encrypts exams and prepares everything for moodle upload.
Attention: contents of folder 'out' will be deleted in the beginning!
Options:
-h, --help show this help text
-i, --in input folder with PDFs. Default: ./pdfs
-c, --csv Moodle grading CSV file, needed to construct folder names
for moodle
-o, --out output folder containing passwords.csv and
moodle_feedbacks.zip. Default: ./out
-p, --password sets global password. Default: empty, such that each PDF
gets custom password
-e, --cores number of cores for watermarking. Default: 1
-d, --dpi dpi parameter for pdf to image conversion. Default: 250
-q, --quality quality parameter for jpeg. Default: 25
-t, --tmp tmp folder. Default: ./tmp
''')
parser.add_argument("-i", "--infolder", default="./pdfs",
help="Input folder with PDFs. Default: ./pdfs")
parser.add_argument("-c", "--csv", default="Bewertungen.csv",
help="Moodle grading CSV file, needed to construct " +
"folder names for moodle zip")
parser.add_argument("-o", "--outfolder", default="./out",
help="output folder containing passwords.csv and " +
"moodle_feedbacks.zip. Default: ./out")
parser.add_argument("-e", "--cores", default="2",
help="Number of cores for parallel processing. " +
"Default: 2")
parser.add_argument("-p", "--password", default="",
help="sets global password. Default: empty, " +
"such that each PDF gets custom password")
parser.add_argument("-d", "--dpi", default="250",
help="DPI parameter for pdf to image conversion. " +
"Default: 250")
parser.add_argument("-t", "--tmp", default="./tmp",
help="tmp folder. Default: ./tmp/")
parser.add_argument("-u", "--supinfolder", default="./supplements",
help="Input folder with sample solutions. Default: ./supplements")
parser.add_argument("-w", "--sup", default="0",
help="Flag for watermarking sample solutions. 0 means no 1 means yes. Default: 0")
parser.add_argument("-x", "--zip", default="0",
help="Input zip file. Default: 0")
args = parser.parse_args(args)
args = _parser.parse_args(args)
infolder = args.infolder
csv = args.csv
outfolder = args.outfolder
......@@ -65,31 +78,46 @@ Options:
dpi = args.dpi
tmp = args.tmp
password = args.password
sup = int(args.sup)
supinfolder = args.supinfolder
watermark_supp = args.supp
supinfolder = args.suppinfolder
inzip = args.zip
csv_enc = args.csvenc
csv_delim = args.csvdelim
csv_quote = args.csvquote
starttime = time.time()
# Unzip submissions if provided zip archive
if inzip != "0" :
if inzip != "0":
if not os.path.exists(infolder):
os.makedirs(infolder)
preparepdf.main(['--in', inzip, '--out', infolder,'--csv', csv])
preparepdf.main([
inzip, csv, infolder,
'--csvenc', csv_enc, '--csvdelim', csv_delim,
'--csvquote', csv_quote])
# Watermarking
watermark_outfolder = os.path.join(tmp, 'pdfs_watermarked')
if not os.path.exists(watermark_outfolder):
os.makedirs(watermark_outfolder)
watermark.main(['--in', infolder, '--out', watermark_outfolder,
'--cores', cores, '--dpi', dpi])
if sup == 1:
watermark.main([
infolder, watermark_outfolder,
'--cores', cores, '--dpi', dpi])
if watermark_supp:
supoutfolder = os.path.join(tmp, 'supplements_out')
if not os.path.exists(supoutfolder):
os.makedirs(supoutfolder)
supplements.main(['--in', supinfolder, '--out', supoutfolder])
watermark.main(['--in', supoutfolder, '--out', watermark_outfolder,
'--cores', cores, '--dpi', dpi])
supplements.main([
supinfolder, csv, supoutfolder,
'--csvenc', csv_enc, '--csvdelim', csv_delim,
'--csvquote', csv_quote
])
watermark.main([supoutfolder, watermark_outfolder,
'--cores', cores, '--dpi', dpi])
# Encryption
enc_out = os.path.join(tmp, 'pdfs_encrypted')
......@@ -97,7 +125,7 @@ Options:
os.makedirs(enc_out)
passwordcsv = os.path.join(outfolder, 'passwords.csv')
encrypt.main(['--in', watermark_outfolder, '--out', enc_out,
encrypt.main([watermark_outfolder, enc_out,
'--passwordout', passwordcsv, '--password', password])
# ZIP Archive preparation process
......@@ -105,8 +133,11 @@ Options:
moodle_tmp = os.path.join(tmp, 'tmp')
if not os.path.exists(moodle_tmp):
os.makedirs(moodle_tmp)
preparemoodle.main(['--in', enc_out, '--csv', csv, '--tmp', moodle_tmp,
'--out', moodle_out])
preparemoodle.main([
enc_out, csv, moodle_out,
'--tmp', moodle_tmp, '--csvenc', csv_enc, '--csvdelim', csv_delim,
'--csvquote', csv_quote])
endtime = time.time()
print(f'\nTotal time taken: {endtime-starttime:.2f}s\n')
......
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd
<header>
.. toctree::
:maxdepth: 3
.. autoprogram:: <file>:_parser
:prog: <file>.py
API
===
.. automodule:: <file>
:members:
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment