Merge branch 'ad_automate_input_pdf' into 'master'

Input automation

See merge request !26

Bewertungen.csv

 ID,"Vollständiger Name",Matrikelnummer,Status,Bewertung,"Status des Bewertungsworkflows","Bewertung kann geändert werden","Zuletzt geändert (Abgabe)","Zuletzt geändert (Bewertung)","Feedback als Kommentar"
-Teilnehmer/in436452,"Nachname, Vorname",123456,"Zur Bewertung abgegeben - Unbewertet",,Unbewertet,Ja,"Montag, 18. Mai 2020, 10:11",-,
-Teilnehmer/in238289,"Noch, Jemand",987654,"Keine Abgabe - Unbewertet",,Unbewertet,Ja,-,"Samstag, 16. Mai 2020, 12:15",-,
+Teilnehmer/in107,"Four, Student",123456,"Zur Bewertung abgegeben - Unbewertet",,Unbewertet,Ja,"Montag, 18. Mai 2020, 10:11",-,
+Teilnehmer/in106,"One, Student",123457,"Keine Abgabe - Unbewertet",,Unbewertet,Ja,-,"Samstag, 16. Mai 2020, 12:15",-,
+Teilnehmer/in105,"Three, Student",125412,"Keine Abgabe - Unbewertet",,Unbewertet,Ja,-,"Samstag, 16. Mai 2020, 12:15",-,
\ No newline at end of file

README.md

@@ -23,11 +23,10 @@ Exemplary outputs can be downloaded:
 ### Prerequisites
 
 * **Create and setup Moodle**
-  * In your Moodle course room, create an `assign` module following this [guideline](https://help.itc.rwth-aachen.de/service/8d9eb2f36eea4fcaa9abd0e1ca008b22/article/0cfca4212fef4712ad2d432ac83eaf3e)
-  * Download the grading table `Bewertungen.csv` from Moodle via: `Alle Angaben anzeigen` → `Bewertungsvorgang` → `Bewertungstabelle  herunterladen`* **Create and setup Moodle**
   * In your Moodle course room, create an `assign` module following this [guideline](https://help.itc.rwth-aachen.de/service/8d9eb2f36eea4fcaa9abd0e1ca008b22/article/0cfca4212fef4712ad2d432ac83eaf3e)
   * Download the grading table `Bewertungen.csv` from Moodle via: `Alle Angaben anzeigen` → `Bewertungsvorgang` → `Bewertungstabelle  herunterladen`
 
+
 * **Create PDFs corresponding to each exam**
   * Scan the exams and save the scans as PDFs (each page should be A4). For most copy machines, you can save an A3 scan (double page of an exam) as two A4 pages.
   * The filename of each PDF should start with the student's matriculation number (e.g. `123456_Nachname.pdf`).
@@ -35,6 +34,10 @@ Exemplary outputs can be downloaded:
   ** OR: Download submission zip file**
   * Download the submission zip file from (Assignment Main Page->View all submissions->Download all submissions)
 
+
+* **OR: Download submission zip file**
+  * Download the submission zip file from (Assignment Main Page->View all submissions->Download all submissions)
+
 * **Optional: Create Sample Solutions (Refer [here](https://git.rwth-aachen.de/rwthmoodle/exam-scan/-/issues/3))**
   * Scan the sample solutions and save the scans as PDFs (each page should be A4). For most copy machines, you can save an A3 scan (double page of an exam) as two A4 pages.
   * Place all PDFs in a folder, e.g. `supplements`.

batch.py

@@ -3,6 +3,8 @@ import sys
 import time
 import argparse
 
+import preparepdf
+import supplements
 import watermark
 import encrypt
 import preparemoodle
@@ -47,6 +49,13 @@ Options:
                         "Default: 250")
     parser.add_argument("-t", "--tmp", default="./tmp",
                         help="tmp folder. Default: ./tmp/")
+    parser.add_argument("-u", "--supinfolder", default="./supplements",
+                        help="Input folder with sample solutions. Default: ./supplements")
+    parser.add_argument("-w", "--sup", default="0",
+                        help="Flag for watermarking sample solutions. 0 means no 1 means yes. Default: 0")
+    parser.add_argument("-x", "--zip", default="0",
+                        help="Input zip file. Default: 0")
+
 
     args = parser.parse_args(args)
     infolder = args.infolder
@@ -56,15 +65,31 @@ Options:
     dpi = args.dpi
     tmp = args.tmp
     password = args.password
+    sup = int(args.sup)
+    supinfolder = args.supinfolder
+    inzip = args.zip
 
     starttime = time.time()
 
+    # Unzip submissions if provided zip archive
+    if inzip != "0" :
+        if not os.path.exists(infolder):
+            os.makedirs(infolder)
+        preparepdf.main(['--in', inzip, '--out', infolder,'--csv', csv])
+    
     # Watermarking
     watermark_outfolder = os.path.join(tmp, 'pdfs_watermarked')
     if not os.path.exists(watermark_outfolder):
         os.makedirs(watermark_outfolder)
     watermark.main(['--in', infolder, '--out', watermark_outfolder,
                     '--cores', cores, '--dpi', dpi])
+    if sup == 1:
+        supoutfolder = os.path.join(tmp, 'supplements_out')
+        if not os.path.exists(supoutfolder):
+            os.makedirs(supoutfolder)
+        supplements.main(['--in', supinfolder, '--out', supoutfolder])
+        watermark.main(['--in', supoutfolder, '--out', watermark_outfolder,
+                    '--cores', cores, '--dpi', dpi])
 
     # Encryption
     enc_out = os.path.join(tmp, 'pdfs_encrypted')

preparepdf.py

+import sys  # get arguments from command line
+import os  # path listing/manipulation/...
+import time  # keep track of time
+import argparse  # handle command line arguments
+import shutil #unzipping and copying files
+import re # pattern matching
+import csv # opening grading worksheet csv
+
+def main(args):
+    """Transfer PDF files from zip file containing all submissions into user provided folder following exam scan naming convention
+
+    1) files are extracted from user-provided zip file location eg: ./all_submissions.zip
+    2) Scan through extracted folder for PDF files. Only 1 PDF file/student is accepted.
+    3) Matriculation number and last name of student is fetched from grading worksheet
+    4) PDFs from extracted folder are renamed according to convention and placed in user provided outfolder
+    """
+
+    # Argument handling
+    parser = argparse.ArgumentParser(description='''
+      Zip file, provided with parameter inzip, containg all submissions of an assignment is extracted, 
+      renamed according to convention and placed in folder provided with prameter --outfolder
+      ''')
+    parser.add_argument("-o", "--outfolder", default="./pdfs",
+        help="Output folder with PDFs followingname schema. Default: ./pdfs")
+    parser.add_argument("-i", "--inzip", default="0",
+        help="Input zip file. Default: 0")
+    parser.add_argument("-c", "--csv", default="./Bewertungen.csv",
+        help="Moodle grading CSV file, needed to construct the folder names. Default: ./Bewertungen.csv")
+    
+
+    args = parser.parse_args(args)
+    inzip = args.inzip
+    outfolder = args.outfolder
+    csvfilename = args.csv
+
+    if inzip == "0" or not(inzip.lower().endswith(('.zip'))):
+        print ('\n***ERROR*** Not a suitable zip file. The script cannot proceed')
+        return
+    else:
+        print('\nExtracting files from '+inzip+' ...')
+        try:
+            extracted_folder = os.path.splitext(inzip)[0]
+            shutil.unpack_archive(inzip, extracted_folder) #unzip file
+        except:
+            print('\n***ERROR*** Something went wrong. Check if you have given the correct name and path for the zip file')
+            return
+
+    with open(csvfilename, newline='') as csvfile:
+        matnums = []
+        moodleids={}
+        reader = csv.reader(csvfile, delimiter=',', quotechar='"')
+        next(reader)  # skip header CSV line
+        for row in reader:
+                # Parse required fields from CSV line
+                # Moodle has its own internal ID per participant alongside
+                # matriculation number
+            moodleid = row[0]
+            moodleid = moodleid.replace("Teilnehmer/in", "")  # German
+            moodleid = moodleid.replace("Participant ", "")  # English
+            
+            name = row[1]  # Lastname, Firstname
+            matnum = row[2]  # matriculation number
+            matnums.append(matnum)  # save matriculation number for later
+            moodleids[moodleid] = matnum
+            pattern = '.*_'+moodleid+'_.*'
+            folder_lists = [folder for folder in os.listdir(extracted_folder) if re.compile(pattern).match(folder)]
+            for folder in folder_lists:
+                print('\n************** For Matriculation number '+matnum+' ****************\n')
+                unsupported_files = []
+                for dirpath, dirnames, filenames in os.walk(os.path.join(extracted_folder,folder)):
+                    if(len([f for f in filenames if f.endswith(".pdf")])>1): # only 1 file per student is allowed
+                        print('Multiple PDF files found in submission. Each student should submit only 1 PDF file')
+                        break
+                    elif (len([f for f in filenames if f.endswith(".pdf")])==0): 
+                        if not dirnames:
+                            print('No PDFs were submitted')# No PDF found in a student's submission
+                        else: #deals if students uploaded a folder instead of a file
+                            continue
+                    else:
+                        for filename in [f for f in filenames if f.endswith(".pdf")]: # renames and copies PDF to outfolder
+                            shutil.copy(os.path.join(dirpath,filename),os.path.join(outfolder,matnum+'_'+name.split(',')[0]+'_'+os.path.splitext(filename)[0]+'.pdf'))
+                            print(filename+' is renamed to '+matnum+'_'+name.split(',')[0]+'.pdf and placed in '+outfolder )
+                for dirpath, dirnames, filenames in os.walk(os.path.join(extracted_folder,folder)):
+                    for filename in [f for f in filenames if not f.endswith(".pdf")]:
+                        unsupported_files.append(filename)
+
+                if unsupported_files: #Lists all non-PDF files found in a student's submission
+                    print('This script only supports PDF files. Hence the below files cannot be handled:')
+                    for filename in unsupported_files:
+                        print('- '+filename)
+        
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

supplements.py

@@ -123,10 +123,11 @@ Files in output folder {} will be overwritten during this process.
     """.format("\n- ".join(supp_files),  output_dir))
 
     # Create prefixes
+    print(pdf_dir)
     if pdf_dir != "":  # Take prefixes from pdf directory
         pdf_folder = os.listdir(pdf_dir)
         pdf_files = [_ for _ in pdf_folder
-                     if _.endswith(".pdf") and utils.check_matnum(_[0:6])]
+                     if _.endswith(".pdf") and utils.check_matnum(_.split('_', 1)[0])]
         prefixes = []
         for pdf_file in pdf_files:
             prefix = os.path.splitext(pdf_file)[0]  # take file name as prefix

utils/matnum.py

@@ -32,7 +32,7 @@ def check_matnum(matnum):
     Returns:
         bool: valid
     """
-    return len(matnum) == 6 and matnum.isdigit()
+    return (len(matnum) == 6 or len(matnum) == 5) and matnum.isdigit()
 
 
 def get_matnum(s):

watermark.py

@@ -247,7 +247,7 @@ def main(args):
     starttime = time.time()
     pdf_folder = os.listdir(infolder)
     pdf_files = [_ for _ in pdf_folder
-                 if _.endswith(".pdf") and utils.check_matnum(_[0:6])]
+                 if _.endswith(".pdf") and utils.check_matnum(_.split('_', 1)[0])]
     print("""
 Available PDFs to be watermarked:
 - {}