Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
E
exam-scan
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Wiki
Requirements
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
RWTHmoodle
exam-scan
Commits
fc93772e
Commit
fc93772e
authored
4 years ago
by
Christian Rohlfing
Browse files
Options
Downloads
Plain Diff
Merge branch 'ad_iss8' of git.rwth-aachen.de:rwthmoodle/exam-scan into ad_iss8
parents
41c22a29
3fff639c
Branches
Branches containing commit
Tags
v3.0.0
Tags containing commit
1 merge request
!39
Automated documentation build
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
preparemoodle.py
+112
-49
112 additions, 49 deletions
preparemoodle.py
with
112 additions
and
49 deletions
preparemoodle.py
+
112
−
49
View file @
fc93772e
...
...
@@ -5,27 +5,102 @@ import time
import
shutil
# copyfile, make_archive
import
argparse
# argument parsing
import
sys
import
zipfile
import
utils.moodle
as
moodle
import
utils.matnum
as
matnum_utils
import
zipfile
def
zip_folders
(
base_folder
,
zip_file
,
size_limit
):
"""
Zip folders in base folder. If size limit is exceeded, create next zip
file until all folders are zipped.
Args:
base_folder (str): path of base folder
zip_file (str): path of zip file
size_limit (int): size limit
def
bytesto
(
bytes
,
to
,
bsize
=
1024
):
"""
convert bytes to megabytes, etc.
sample code:
print(
'
mb=
'
+ str(bytesto(314575262000000,
'
m
'
)))
sample output:
mb= 300002347.946
Returns:
int: number of zip files
"""
a
=
{
'
k
'
:
1
,
'
m
'
:
2
,
'
g
'
:
3
,
'
t
'
:
4
,
'
p
'
:
5
,
'
e
'
:
6
}
r
=
float
(
bytes
)
for
i
in
range
(
a
[
to
]):
r
=
r
/
bsize
# Initialize
zip_file_base
=
os
.
path
.
splitext
(
zip_file
)[
0
]
total_compress_size
=
0
zip_cnt
=
0
zf
=
None
# Iterate over folders
all_folders
=
os
.
listdir
(
base_folder
)
num_folders
=
len
(
all_folders
)
for
cnt
,
folder
in
enumerate
(
all_folders
):
# Measure uncompressed folder path
folder_path
=
os
.
path
.
join
(
base_folder
,
folder
)
folder_size
=
get_folder_size
(
folder_path
)
folder_size
/=
1024
**
2
# conversion from bytes to MiB
# If size_limit reached, create new zip file
if
total_compress_size
+
folder_size
>
size_limit
or
zf
is
None
:
# File name
zip_cnt
+=
1
if
zf
is
not
None
:
zf
.
close
()
# Close previous zip file
zip_file
=
"
{zip}_{cnt}.zip
"
.
format
(
zip
=
zip_file_base
,
cnt
=
zip_cnt
)
# Reset counters
total_compress_size
=
0
file_cnt
=
0
# Open (new) zip file
zf
=
zipfile
.
ZipFile
(
zip_file
,
mode
=
'
w
'
,
compression
=
zipfile
.
ZIP_DEFLATED
)
# Loop over files in current folder
last_file_cnt
=
file_cnt
for
f
in
os
.
listdir
(
folder_path
):
# Add file to zip file
zf
.
write
(
os
.
path
.
join
(
folder_path
,
f
),
arcname
=
os
.
path
.
join
(
folder
,
f
))
file_cnt
+=
1
# Get compressed size of folder
folder_compress_size
=
sum
(
[
_
.
compress_size
for
_
in
zf
.
filelist
[
last_file_cnt
:]])
folder_compress_size
/=
1024
**
2
# conversion from bytes to MiB
total_compress_size
+=
folder_compress_size
# Print for-loop progress
if
not
(
cnt
%
max
(
1
,
round
(
num_folders
/
10
))):
print
(
"
.
"
,
sep
=
'
'
,
end
=
''
,
flush
=
True
)
# Clean up
zf
.
close
()
print
(
"
done.
"
)
return
zip_cnt
def
get_folder_size
(
path
):
"""
Get size in bytes of folder
Args:
path (str): path of folder
Returns:
int: number of bytes
"""
total
=
0
for
entry
in
os
.
scandir
(
path
):
if
entry
.
is_file
():
size
=
entry
.
stat
().
st_size
elif
entry
.
is_dir
():
size
=
get_folder_size
(
entry
.
path
)
total
+=
size
return
total
return
(
round
(
r
,
2
))
def
sanity_check
(
matnums_csv
,
matnums_folder
):
"""
Check two cases for sanity:
...
...
@@ -69,8 +144,11 @@ def main(args):
Attention: zip-archive
'
out
'
will be overwritten in the following!
'''
)
parser
.
add_argument
(
"
infolder
"
,
help
=
"
Input folder with PDFs.
"
)
parser
.
add_argument
(
"
csv
"
,
help
=
"
Moodle grading sheet.
"
)
parser
.
add_argument
(
"
infolder
"
,
help
=
"
Input folder with PDFs.
"
)
parser
.
add_argument
(
"
-c
"
,
"
--csv
"
,
default
=
"
./Bewertungen.csv
"
,
help
=
"
Moodle grading sheet. Default: ./Bewertungen.csv
"
)
parser
.
add_argument
(
"
--csvdelim
"
,
default
=
"
,
"
,
help
=
"
CSV delimiter. Default:
'
,
'"
)
parser
.
add_argument
(
...
...
@@ -79,16 +157,17 @@ def main(args):
"
--csvenc
"
,
default
=
"
utf-8
"
,
help
=
"
CSV encoding scheme.
"
+
"
Typical encodings:
'
utf-8
'
,
'
utf-8-sig
'
, or
'
cp1252
'
(Windows).
"
+
"
Default:
'
utf-8
'"
)
parser
.
add_argument
(
"
outzip
"
,
help
=
"
Zip archive.
"
)
parser
.
add_argument
(
"
outzip
"
,
help
=
"
Zip archive.
"
)
parser
.
add_argument
(
"
-d
"
,
"
--dry
"
,
action
=
'
store_true
'
,
help
=
"
Flag for dry run.
"
)
parser
.
add_argument
(
"
-t
"
,
"
--tmp
"
,
default
=
"
./tmp
"
,
help
=
"
Temporary folder. Default:./tmp
"
)
parser
.
add_argument
(
"
--nowarn
"
,
action
=
'
store_true
'
,
help
=
"
Disables warnings
"
)
parser
.
add_argument
(
"
--moodlefilesize
"
,
default
=
"
250
"
,
help
=
"
Moodle upload file size in MiB. Default: 250
"
)
parser
.
add_argument
(
"
--nowarn
"
,
action
=
'
store_true
'
,
help
=
"
Disables warnings
"
)
args
=
parser
.
parse_args
(
args
)
infolder
=
args
.
infolder
...
...
@@ -120,7 +199,8 @@ Processing {} students'''.format(num_students))
os
.
remove
(
outzip
)
# Create temporary folder within given temporary directory
if
not
os
.
path
.
isdir
(
tmp_folder
):
if
os
.
path
.
isdir
(
tmp_folder
):
shutil
.
rmtree
(
tmp_folder
)
os
.
mkdir
(
tmp_folder
)
# Parse input folder
...
...
@@ -130,9 +210,6 @@ Processing {} students'''.format(num_students))
allfiles
=
os
.
listdir
(
infolder
)
allfiles
.
sort
()
allpdfs
=
[]
if
(
len
(
allfiles
)
==
0
):
print
(
"""
There are no PDFs in the given directory. Exiting now.
"""
)
return
for
f
in
allfiles
:
if
f
.
lower
().
endswith
(
'
.pdf
'
)
and
matnum_utils
.
starts_with_matnum
(
f
):
allpdfs
.
append
(
f
)
...
...
@@ -184,7 +261,8 @@ Processing {} students'''.format(num_students))
else
:
dryout
.
append
(
"
- {old} -> {new}
"
.
format
(
old
=
pdffile
,
new
=
os
.
path
.
join
(
folder
,
pdffile
)))
.
format
(
old
=
pdffile
,
new
=
os
.
path
.
join
(
folder
,
pdffile
)))
elif
not
no_warn
:
# No PDF found
print
(
"
Warning: PDF for {matnum} (id={id}, name={name}) not found.
"
...
...
@@ -199,38 +277,23 @@ Processing {} students'''.format(num_students))
print
(
"
Found {num_pdf} PDFs (CSV had {num_csv} entries)
"
.
format
(
num_pdf
=
num_found_pdfs
,
num_csv
=
num_students
))
# Sanity check:
# Check for PDFs not reflected in CSV (student not registered in Moodle)
sanity_check
(
matnums_csv
,
matnums_folder
)
# Zip
if
not
dry
:
foldersize
=
0
count
=
1
z
=
zipfile
.
ZipFile
(
os
.
path
.
splitext
(
outzip
)[
0
]
+
str
(
count
)
+
"
.zip
"
,
"
w
"
)
for
dirpath
,
dirnames
,
filenames
in
os
.
walk
(
tmp_folder
):
for
file
in
filenames
:
file_path
=
os
.
path
.
join
(
dirpath
,
file
)
if
not
os
.
path
.
islink
(
file_path
):
foldersize
+=
os
.
path
.
getsize
(
file_path
)
if
bytesto
(
foldersize
,
'
m
'
)
<
size_limit
:
z
.
write
(
file_path
,
os
.
path
.
join
(
os
.
path
.
relpath
(
file_path
,
tmp_folder
),
file
))
os
.
remove
(
file_path
)
else
:
print
(
"
Preparing zip file
"
+
str
(
count
))
z
.
close
()
print
(
'
Zip archive is stored at {}
'
.
format
(
os
.
path
.
splitext
(
outzip
)[
0
]
+
str
(
count
)
+
"
.zip
"
))
count
+=
1
foldersize
=
0
z
=
zipfile
.
ZipFile
(
os
.
path
.
splitext
(
outzip
)[
0
]
+
str
(
count
)
+
"
.zip
"
,
"
w
"
)
z
.
write
(
file_path
,
os
.
path
.
join
(
os
.
path
.
relpath
(
file_path
,
tmp_folder
),
file
))
os
.
remove
(
file_path
)
# Delete temporary folder
print
(
"
Zipping
"
)
zip_cnt
=
zip_folders
(
base_folder
=
tmp_folder
,
zip_file
=
outzip
,
size_limit
=
size_limit
)
# Remove temporary folder
shutil
.
rmtree
(
tmp_folder
)
# Print status
print
(
"
{cnt} zip archives are stored ({zip}*)
"
.
format
(
cnt
=
zip_cnt
,
zip
=
os
.
path
.
splitext
(
outzip
)[
0
]))
# Print dry run results
else
:
dryout
.
sort
()
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment