Skip to content
Snippets Groups Projects
Commit 1661db24 authored by Amrita Deb's avatar Amrita Deb
Browse files

Merge branch 'rohlfing-preparemoodle-zip-size' into 'master'

Fixing issue #6

Closes #6

See merge request !38
parents a270e80b 57f2bdfb
No related branches found
Tags v1.0.1
1 merge request!38Fixing issue #6
Pipeline #457893 passed
......@@ -5,11 +5,103 @@ import time
import shutil # copyfile, make_archive
import argparse # argument parsing
import sys
import zipfile
import utils.moodle as moodle
import utils.matnum as matnum_utils
def zip_folders(base_folder, zip_file, size_limit):
"""Zip folders in base folder. If size limit is exceeded, create next zip
file until all folders are zipped.
Args:
base_folder (str): path of base folder
zip_file (str): path of zip file
size_limit (int): size limit
Returns:
int: number of zip files
"""
# Initialize
zip_file_base = os.path.splitext(zip_file)[0]
total_compress_size = 0
zip_cnt = 0
zf = None
# Iterate over folders
all_folders = os.listdir(base_folder)
num_folders = len(all_folders)
for cnt, folder in enumerate(all_folders):
# Measure uncompressed folder path
folder_path = os.path.join(base_folder, folder)
folder_size = get_folder_size(folder_path)
folder_size /= 1024**2 # conversion from bytes to MiB
# If size_limit reached, create new zip file
if total_compress_size + folder_size > size_limit or zf is None:
# File name
zip_cnt += 1
if zf is not None:
zf.close() # Close previous zip file
zip_file = "{zip}_{cnt}.zip".format(
zip=zip_file_base, cnt=zip_cnt)
# Reset counters
total_compress_size = 0
file_cnt = 0
# Open (new) zip file
zf = zipfile.ZipFile(
zip_file, mode='w', compression=zipfile.ZIP_DEFLATED)
# Loop over files in current folder
last_file_cnt = file_cnt
for f in os.listdir(folder_path):
# Add file to zip file
zf.write(
os.path.join(folder_path, f), arcname=os.path.join(folder, f))
file_cnt += 1
# Get compressed size of folder
folder_compress_size = sum(
[_.compress_size for _ in zf.filelist[last_file_cnt:]])
folder_compress_size /= 1024**2 # conversion from bytes to MiB
total_compress_size += folder_compress_size
# Print for-loop progress
if not (cnt % max(1, round(num_folders/10))):
print(".", sep=' ', end='', flush=True)
# Clean up
zf.close()
print("done.")
return zip_cnt
def get_folder_size(path):
"""Get size in bytes of folder
Args:
path (str): path of folder
Returns:
int: number of bytes
"""
total = 0
for entry in os.scandir(path):
if entry.is_file():
size = entry.stat().st_size
elif entry.is_dir():
size = get_folder_size(entry.path)
total += size
return total
def sanity_check(matnums_csv, matnums_folder):
"""Check two cases for sanity:
- Are there PDF files with no corresponding CSV entries?
......@@ -75,6 +167,9 @@ def main(args):
"-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
parser.add_argument(
"--nowarn", action='store_true', help="Disables warnings")
parser.add_argument(
"--moodlefilesize", default="250",
help="Moodle upload file size in MiB. Default: 250")
args = parser.parse_args(args)
infolder = args.infolder
......@@ -86,6 +181,7 @@ def main(args):
csv_delim = args.csvdelim
csv_quote = args.csvquote
csv_enc = args.csvenc
size_limit = int(args.moodlefilesize) # Moodle upload size limit in MiB
# Print status
starttime = time.time()
......@@ -105,8 +201,9 @@ Processing {} students'''.format(num_students))
os.remove(outzip)
# Create temporary folder within given temporary directory
if not os.path.isdir(tmp_folder):
os.mkdir(tmp_folder)
if os.path.isdir(tmp_folder):
shutil.rmtree(tmp_folder)
os.mkdir(tmp_folder)
# Parse input folder
# Only PDF files are considered with first digits
......@@ -166,7 +263,8 @@ Processing {} students'''.format(num_students))
else:
dryout.append(
"- {old} -> {new}"
.format(old=pdffile, new=os.path.join(folder, pdffile)))
.format(
old=pdffile, new=os.path.join(folder, pdffile)))
elif not no_warn: # No PDF found
print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
......@@ -180,7 +278,6 @@ Processing {} students'''.format(num_students))
print("done.")
print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
.format(num_pdf=num_found_pdfs, num_csv=num_students))
# Sanity check:
# Check for PDFs not reflected in CSV (student not registered in Moodle)
......@@ -188,14 +285,17 @@ Processing {} students'''.format(num_students))
# Zip
if not dry:
# Zip
print("Zipping")
shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmp_folder)
print('Zip archive is stored at {}'.format(outzip))
zip_cnt = zip_folders(
base_folder=tmp_folder, zip_file=outzip, size_limit=size_limit)
# Delete temporary folder
# Remove temporary folder
shutil.rmtree(tmp_folder)
# Print status
print("{cnt} zip archives are stored ({zip}*)"
.format(cnt=zip_cnt, zip=os.path.splitext(outzip)[0]))
# Print dry run results
else:
dryout.sort()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment