Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
aknecht2
Image Harvest
Commits
83c257b1
Commit
83c257b1
authored
May 05, 2016
by
aknecht2
Browse files
Adjusted sql aggregation to add values in chunks of 500 columns.
parent
dcbd56c8
Changes
1
Hide whitespace changes
Inline
Side-by-side
ih/statistics.py
View file @
83c257b1
...
...
@@ -27,6 +27,7 @@ import json
import
traceback
import
shutil
import
subprocess
import
itertools
from
scipy
import
stats
class
Stats
:
...
...
@@ -217,14 +218,18 @@ class Stats:
for
i
,
f
in
enumerate
(
dblist
):
if
os
.
path
.
isfile
(
f
):
conn
=
self
.
_openConnection
(
f
)
cols
=
self
.
_getHeaders
(
"images"
,
conn
)
for
col
in
cols
:
self
.
_addColumn
(
col
,
"images"
)
result
=
conn
.
execute
(
"select pegasusid,"
+
","
.
join
(
cols
)
+
" from images"
)
query
=
"update images set "
+
","
.
join
([
col
+
"=?"
for
col
in
cols
])
+
" where pegasusid=?"
values
=
[
tuple
([
row
[
col
]
for
col
in
cols
]
+
[
row
[
"pegasusid"
]])
for
row
in
result
]
self
.
conn
.
executemany
(
query
,
values
)
self
.
conn
.
commit
()
headers
=
self
.
_getHeaders
(
"images"
,
conn
)
col_set
=
itertools
.
islice
(
headers
,
500
)
while
col_set
:
cols
=
list
(
col_set
)
for
col
in
cols
:
self
.
_addColumn
(
col
,
"images"
)
result
=
conn
.
execute
(
"select pegasusid,"
+
","
.
join
(
cols
)
+
" from images"
)
query
=
"update images set "
+
","
.
join
([
col
+
"=?"
for
col
in
cols
])
+
" where pegasusid=?"
values
=
[
tuple
([
row
[
col
]
for
col
in
cols
]
+
[
row
[
"pegasusid"
]])
for
row
in
result
]
self
.
conn
.
executemany
(
query
,
values
)
self
.
conn
.
commit
()
col_set
=
itertools
.
islice
(
headers
,
500
)
self
.
_closeConnection
(
conn
)
else
:
print
"DB File: '%s' does not exist."
%
(
f
,)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment