Commit 5a0dba64 authored by Christian Rohlfing's avatar Christian Rohlfing
Browse files

Fixing issue #6

parent a270e80b
Pipeline #456131 passed with stage
in 1 minute and 48 seconds
......@@ -10,6 +10,27 @@ import utils.moodle as moodle
import utils.matnum as matnum_utils
def folder_size(path):
total = 0
cumsum = [] # cumulative sum
norm = 1048576 # bytes to MiB
for entry in os.scandir(path):
if entry.is_file():
size = entry.stat().st_size
elif entry.is_dir():
size, _ = folder_size(entry.path)
size *= norm
size /= norm
total += size
if len(cumsum) > 1:
cumsum.append(cumsum[-1]+size)
else:
cumsum.append(size)
return total, cumsum
def sanity_check(matnums_csv, matnums_folder):
"""Check two cases for sanity:
- Are there PDF files with no corresponding CSV entries?
......@@ -75,6 +96,9 @@ def main(args):
"-t", "--tmp", default="./tmp", help="Temporary folder. Default:./tmp")
parser.add_argument(
"--nowarn", action='store_true', help="Disables warnings")
parser.add_argument(
"--moodlefilesize", default="5",
help="Moodle upload file size in MiB. Default: 250")
args = parser.parse_args(args)
infolder = args.infolder
......@@ -86,6 +110,7 @@ def main(args):
csv_delim = args.csvdelim
csv_quote = args.csvquote
csv_enc = args.csvenc
size_limit = int(args.moodlefilesize) # Moodle upload size limit in MiB
# Print status
starttime = time.time()
......@@ -105,8 +130,9 @@ Processing {} students'''.format(num_students))
os.remove(outzip)
# Create temporary folder within given temporary directory
if not os.path.isdir(tmp_folder):
os.mkdir(tmp_folder)
if os.path.isdir(tmp_folder):
shutil.rmtree(tmp_folder)
os.mkdir(tmp_folder)
# Parse input folder
# Only PDF files are considered with first digits
......@@ -180,7 +206,6 @@ Processing {} students'''.format(num_students))
print("done.")
print("Found {num_pdf} PDFs (CSV had {num_csv} entries)"
.format(num_pdf=num_found_pdfs, num_csv=num_students))
# Sanity check:
# Check for PDFs not reflected in CSV (student not registered in Moodle)
......@@ -190,8 +215,53 @@ Processing {} students'''.format(num_students))
if not dry:
# Zip
print("Zipping")
shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmp_folder)
print('Zip archive is stored at {}'.format(outzip))
total_size, cum_size = folder_size(tmp_folder)
if total_size < size_limit:
shutil.make_archive(os.path.splitext(outzip)[0], 'zip', tmp_folder)
print('Zip archive is stored at {}'.format(outzip))
else:
# Create multiple zip files
pkg_cnt = 1
# Intermediate folder
pkg_folder = os.path.join(args.tmp, "to_be_zipped_for_moodle_pkg")
if os.path.isdir(pkg_folder):
shutil.rmtree(pkg_folder)
os.mkdir(pkg_folder)
# Iterate over folders
tmp_folder_contents = os.listdir(tmp_folder)
for folder_cnt, folder in enumerate(tmp_folder_contents):
folder_src_path = os.path.join(tmp_folder, folder)
folder_dst_path = os.path.join(pkg_folder, folder)
# Copy current folder to package folder
shutil.copytree(folder_src_path, folder_dst_path)
# Zip and start new package if next file reaches size_limit
if folder_cnt < len(tmp_folder_contents)-1:
pkg_size = cum_size[folder_cnt+1]-(pkg_cnt-1)*size_limit
else:
# Last folder reached. We have to zip anyway
pkg_size = size_limit + 1
if pkg_size > size_limit:
# Zip package
pkg_file = "{zip}_{cnt:03d}".format(
zip=os.path.splitext(outzip)[0], cnt=pkg_cnt)
shutil.make_archive(pkg_file, 'zip', pkg_folder)
# Clear package
for _ in os.listdir(pkg_folder):
shutil.rmtree(os.path.join(pkg_folder, _))
pkg_cnt += 1
print(
"{cnt} zip archives are stored beginning with {zip}_"
.format(cnt=pkg_cnt-1, zip=os.path.splitext(outzip)[0]))
# Clean up
shutil.rmtree(pkg_folder)
# Delete temporary folder
shutil.rmtree(tmp_folder)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment