Commit f1b53082 authored by Christian Rohlfing's avatar Christian Rohlfing
Browse files

Beautification

parent 5a0dba64
......@@ -5,30 +5,90 @@ import time
import shutil # copyfile, make_archive
import argparse # argument parsing
import sys
import zipfile
import utils.moodle as moodle
import utils.matnum as matnum_utils
def folder_size(path):
def zip_folders(base_folder, zip_file, size_limit):
# Initialize
zip_file_base = os.path.splitext(zip_file)[0]
total_compress_size = 0
zip_cnt = 0
zf = None
# Iterate over folders
all_folders = os.listdir(base_folder)
num_folders = len(all_folders)
for cnt, folder in enumerate(all_folders):
# Measure uncompressed folder path
folder_path = os.path.join(base_folder, folder)
folder_size = get_folder_size(folder_path)
folder_size /= 1024**2 # conversion from bytes to MiB
# If size_limit reached, create new zip file
if total_compress_size + folder_size > size_limit or zf is None:
# File name
zip_cnt += 1
if zf is not None:
zf.close() # Close previous zip file
zip_file = "{zip}_{cnt}.zip".format(
zip=zip_file_base, cnt=zip_cnt)
# Reset counters
total_compress_size = 0
file_cnt = 0
# Open (new) zip file
zf = zipfile.ZipFile(
zip_file, mode='w', compression=zipfile.ZIP_DEFLATED)
# Loop over files in current folder
last_file_cnt = file_cnt
for f in os.listdir(folder_path):
# Add file to zip file
zf.write(
os.path.join(folder_path, f), arcname=os.path.join(folder, f))
file_cnt += 1
# Get compressed size of folder
folder_compress_size = sum(
[_.compress_size for _ in zf.filelist[last_file_cnt:]])
folder_compress_size /= 1024**2 # conversion from bytes to MiB
total_compress_size += folder_compress_size
# Print for-loop progress
if not (cnt % max(1, round(num_folders/10))):
print(".", sep=' ', end='', flush=True)
# Clean up
zf.close()
print("done.")
return zip_cnt
def get_folder_size(path):
"""Get size in bytes of folder
Args:
path (str): path of folder
Returns:
int: number of bytes
"""
total = 0
cumsum = [] # cumulative sum
norm = 1048576 # bytes to MiB
for entry in os.scandir(path):
if entry.is_file():
size = entry.stat().st_size
elif entry.is_dir():
size, _ = folder_size(entry.path)
size *= norm
size = get_folder_size(entry.path)
size /= norm
total += size
if len(cumsum) > 1:
cumsum.append(cumsum[-1]+size)
else:
cumsum.append(size)
return total, cumsum
return total
def sanity_check(matnums_csv, matnums_folder):
......@@ -97,7 +157,7 @@ def main(args):
parser.add_argument(
"--nowarn", action='store_true', help="Disables warnings")
parser.add_argument(
"--moodlefilesize", default="5",
"--moodlefilesize", default="250",
help="Moodle upload file size in MiB. Default: 250")
args = parser.parse_args(args)
......@@ -192,7 +252,8 @@ Processing {} students'''.format(num_students))
else:
dryout.append(
"- {old} -> {new}"
.format(old=pdffile, new=os.path.join(folder, pdffile)))
.format(
old=pdffile, new=os.path.join(folder, pdffile)))
elif not no_warn: # No PDF found
print("Warning: PDF for {matnum} (id={id}, name={name}) not found."
......@@ -213,59 +274,17 @@ Processing {} students'''.format(num_students))
# Zip
if not dry:
# Zip
print("Zipping")
total_size, cum_size = folder_size(tmp_folder)
if total_size < size_limit:
shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmp_folder)
print('Zip archive is stored at {}'.format(outzip))
else:
# Create multiple zip files
pkg_cnt = 1
# Intermediate folder
pkg_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle_pkg")
if os.path.isdir(pkg_folder):
shutil.rmtree(pkg_folder)
os.mkdir(pkg_folder)
# Iterate over folders
tmp_folder_contents = os.listdir(tmp_folder)
for folder_cnt, folder in enumerate(tmp_folder_contents):
folder_src_path = os.path.join(tmp_folder, folder)
folder_dst_path = os.path.join(pkg_folder, folder)
# Copy current folder to package folder
shutil.copytree(folder_src_path, folder_dst_path)
# Zip and start new package if next file reaches size_limit
if folder_cnt < len(tmp_folder_contents)-1:
pkg_size = cum_size[folder_cnt+1]-(pkg_cnt-1)*size_limit
else:
# Last folder reached. We have to zip anyway
pkg_size = size_limit + 1
if pkg_size > size_limit:
# Zip package
pkg_file = "{zip}_{cnt:03d}".format(
zip=os.path.splitext(outzip)[0], cnt=pkg_cnt)
shutil.make_archive(pkg_file, 'zip', pkg_folder)
zip_cnt = zip_folders(
base_folder=tmp_folder, zip_file=outzip, size_limit=size_limit)
# Clear package
for _ in os.listdir(pkg_folder):
shutil.rmtree(os.path.join(pkg_folder, _))
pkg_cnt += 1
print(
"{cnt} zip archives are stored beginning with {zip}_"
.format(cnt=pkg_cnt-1, zip=os.path.splitext(outzip)[0]))
# Clean up
shutil.rmtree(pkg_folder)
# Delete temporary folder
# Remove temporary folder
shutil.rmtree(tmp_folder)
# Print status
print("{cnt} zip archives are stored ({zip}*)"
.format(cnt=zip_cnt, zip=os.path.splitext(outzip)[0]))
# Print dry run results
else:
dryout.sort()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment