elasticizer: updated scroll time to be longer than the timeouts every 200000 articles (so 20m scroll time, 900s (15m) timeout)

master
Erik de Vries 6 years ago
parent 4ad5357e15
commit a3c3651c79

@ -37,10 +37,10 @@ elasticizer <- function(query, src = T, index = 'maml', es_pwd = .rs.askForPassw
} }
# Get all results - one approach is to use a while loop # Get all results - one approach is to use a while loop
if (src == T) { if (src == T) {
res <- Search(index = index, time_scroll="5m",body = query, size = 1000, raw=T) res <- Search(index = index, time_scroll="20m",body = query, size = 1000, raw=T)
} }
if (src == F) { if (src == F) {
res <- Search(index = index, time_scroll="5m",body = query, size = 1000, raw=T, source = F) res <- Search(index = index, time_scroll="20m",body = query, size = 1000, raw=T, source = F)
} }
json <- fromJSON(res) json <- fromJSON(res)
if (json$hits$total == 0) { if (json$hits$total == 0) {
@ -55,7 +55,7 @@ elasticizer <- function(query, src = T, index = 'maml', es_pwd = .rs.askForPassw
update(out, localhost = localhost, ...) update(out, localhost = localhost, ...)
} }
while(hits != 0){ while(hits != 0){
res <- scroll(json$`_scroll_id`, time_scroll="5m", raw=T) res <- scroll(json$`_scroll_id`, time_scroll="20m", raw=T)
json <- fromJSON(res) json <- fromJSON(res)
hits <- length(json$hits$hits) hits <- length(json$hits$hits)
if(hits > 0) { if(hits > 0) {

Loading…
Cancel
Save