Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
E
exam-scan
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Requirements
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
RWTHmoodle
exam-scan
Commits
1661db24
Commit
1661db24
authored
3 years ago
by
Amrita Deb
Browse files
Options
Downloads
Plain Diff
Merge branch 'rohlfing-preparemoodle-zip-size' into 'master'
Fixing issue
#6
Closes
#6
See merge request
!38
parents
a270e80b
57f2bdfb
No related branches found
Branches containing commit
Tags
v1.0.1
1 merge request
!38
Fixing issue #6
Pipeline
#457893
passed
3 years ago
Stage: build
Stage: test
Changes
1
Pipelines
3
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
preparemoodle.py
+108
-8
108 additions, 8 deletions
preparemoodle.py
with
108 additions
and
8 deletions
preparemoodle.py
+
108
−
8
View file @
1661db24
...
...
@@ -5,11 +5,103 @@ import time
import
shutil
# copyfile, make_archive
import
argparse
# argument parsing
import
sys
import
zipfile
import
utils.moodle
as
moodle
import
utils.matnum
as
matnum_utils
def
zip_folders
(
base_folder
,
zip_file
,
size_limit
):
"""
Zip folders in base folder. If size limit is exceeded, create next zip
file until all folders are zipped.
Args:
base_folder (str): path of base folder
zip_file (str): path of zip file
size_limit (int): size limit
Returns:
int: number of zip files
"""
# Initialize
zip_file_base
=
os
.
path
.
splitext
(
zip_file
)[
0
]
total_compress_size
=
0
zip_cnt
=
0
zf
=
None
# Iterate over folders
all_folders
=
os
.
listdir
(
base_folder
)
num_folders
=
len
(
all_folders
)
for
cnt
,
folder
in
enumerate
(
all_folders
):
# Measure uncompressed folder path
folder_path
=
os
.
path
.
join
(
base_folder
,
folder
)
folder_size
=
get_folder_size
(
folder_path
)
folder_size
/=
1024
**
2
# conversion from bytes to MiB
# If size_limit reached, create new zip file
if
total_compress_size
+
folder_size
>
size_limit
or
zf
is
None
:
# File name
zip_cnt
+=
1
if
zf
is
not
None
:
zf
.
close
()
# Close previous zip file
zip_file
=
"
{zip}_{cnt}.zip
"
.
format
(
zip
=
zip_file_base
,
cnt
=
zip_cnt
)
# Reset counters
total_compress_size
=
0
file_cnt
=
0
# Open (new) zip file
zf
=
zipfile
.
ZipFile
(
zip_file
,
mode
=
'
w
'
,
compression
=
zipfile
.
ZIP_DEFLATED
)
# Loop over files in current folder
last_file_cnt
=
file_cnt
for
f
in
os
.
listdir
(
folder_path
):
# Add file to zip file
zf
.
write
(
os
.
path
.
join
(
folder_path
,
f
),
arcname
=
os
.
path
.
join
(
folder
,
f
))
file_cnt
+=
1
# Get compressed size of folder
folder_compress_size
=
sum
(
[
_
.
compress_size
for
_
in
zf
.
filelist
[
last_file_cnt
:]])
folder_compress_size
/=
1024
**
2
# conversion from bytes to MiB
total_compress_size
+=
folder_compress_size
# Print for-loop progress
if
not
(
cnt
%
max
(
1
,
round
(
num_folders
/
10
))):
print
(
"
.
"
,
sep
=
'
'
,
end
=
''
,
flush
=
True
)
# Clean up
zf
.
close
()
print
(
"
done.
"
)
return
zip_cnt
def
get_folder_size
(
path
):
"""
Get size in bytes of folder
Args:
path (str): path of folder
Returns:
int: number of bytes
"""
total
=
0
for
entry
in
os
.
scandir
(
path
):
if
entry
.
is_file
():
size
=
entry
.
stat
().
st_size
elif
entry
.
is_dir
():
size
=
get_folder_size
(
entry
.
path
)
total
+=
size
return
total
def
sanity_check
(
matnums_csv
,
matnums_folder
):
"""
Check two cases for sanity:
- Are there PDF files with no corresponding CSV entries?
...
...
@@ -75,6 +167,9 @@ def main(args):
"
-t
"
,
"
--tmp
"
,
default
=
"
./tmp
"
,
help
=
"
Temporary folder. Default:./tmp
"
)
parser
.
add_argument
(
"
--nowarn
"
,
action
=
'
store_true
'
,
help
=
"
Disables warnings
"
)
parser
.
add_argument
(
"
--moodlefilesize
"
,
default
=
"
250
"
,
help
=
"
Moodle upload file size in MiB. Default: 250
"
)
args
=
parser
.
parse_args
(
args
)
infolder
=
args
.
infolder
...
...
@@ -86,6 +181,7 @@ def main(args):
csv_delim
=
args
.
csvdelim
csv_quote
=
args
.
csvquote
csv_enc
=
args
.
csvenc
size_limit
=
int
(
args
.
moodlefilesize
)
# Moodle upload size limit in MiB
# Print status
starttime
=
time
.
time
()
...
...
@@ -105,8 +201,9 @@ Processing {} students'''.format(num_students))
os
.
remove
(
outzip
)
# Create temporary folder within given temporary directory
if
not
os
.
path
.
isdir
(
tmp_folder
):
os
.
mkdir
(
tmp_folder
)
if
os
.
path
.
isdir
(
tmp_folder
):
shutil
.
rmtree
(
tmp_folder
)
os
.
mkdir
(
tmp_folder
)
# Parse input folder
# Only PDF files are considered with first digits
...
...
@@ -166,7 +263,8 @@ Processing {} students'''.format(num_students))
else
:
dryout
.
append
(
"
- {old} -> {new}
"
.
format
(
old
=
pdffile
,
new
=
os
.
path
.
join
(
folder
,
pdffile
)))
.
format
(
old
=
pdffile
,
new
=
os
.
path
.
join
(
folder
,
pdffile
)))
elif
not
no_warn
:
# No PDF found
print
(
"
Warning: PDF for {matnum} (id={id}, name={name}) not found.
"
...
...
@@ -180,7 +278,6 @@ Processing {} students'''.format(num_students))
print
(
"
done.
"
)
print
(
"
Found {num_pdf} PDFs (CSV had {num_csv} entries)
"
.
format
(
num_pdf
=
num_found_pdfs
,
num_csv
=
num_students
))
# Sanity check:
# Check for PDFs not reflected in CSV (student not registered in Moodle)
...
...
@@ -188,14 +285,17 @@ Processing {} students'''.format(num_students))
# Zip
if
not
dry
:
# Zip
print
(
"
Zipping
"
)
shutil
.
make_archive
(
os
.
path
.
splitext
(
outzip
)[
0
],
'
zip
'
,
tm
p_folder
)
print
(
'
Zip archive is stored at {}
'
.
format
(
outzip
)
)
zip_cnt
=
zi
p_folder
s
(
base_folder
=
tmp_folder
,
zip_file
=
outzip
,
size_limit
=
size_limit
)
#
Delet
e temporary folder
#
Remov
e temporary folder
shutil
.
rmtree
(
tmp_folder
)
# Print status
print
(
"
{cnt} zip archives are stored ({zip}*)
"
.
format
(
cnt
=
zip_cnt
,
zip
=
os
.
path
.
splitext
(
outzip
)[
0
]))
# Print dry run results
else
:
dryout
.
sort
()
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment