|
|
@ -39,6 +39,7 @@ dupe_detect <- function(row, grid, cutoff_lower, cutoff_upper = 1, es_pwd, es_su
|
|
|
|
diag(simil) <- NA
|
|
|
|
diag(simil) <- NA
|
|
|
|
duplicates <- which(simil >= cutoff_lower & simil <= cutoff_upper, arr.ind = TRUE)
|
|
|
|
duplicates <- which(simil >= cutoff_lower & simil <= cutoff_upper, arr.ind = TRUE)
|
|
|
|
duplicates <- cbind(duplicates, rowid= rownames(duplicates))
|
|
|
|
duplicates <- cbind(duplicates, rowid= rownames(duplicates))
|
|
|
|
|
|
|
|
if (length(duplicates) > 0) {
|
|
|
|
rownames(duplicates) <- seq(1:length(rownames(duplicates)))
|
|
|
|
rownames(duplicates) <- seq(1:length(rownames(duplicates)))
|
|
|
|
df <- as.data.frame(duplicates, make.names = NA, stringsAsFactors = F) %>%
|
|
|
|
df <- as.data.frame(duplicates, make.names = NA, stringsAsFactors = F) %>%
|
|
|
|
# bind_cols(colid = colnames(simil)[.['col']]) %>%
|
|
|
|
# bind_cols(colid = colnames(simil)[.['col']]) %>%
|
|
|
@ -55,10 +56,11 @@ dupe_detect <- function(row, grid, cutoff_lower, cutoff_upper = 1, es_pwd, es_su
|
|
|
|
dupe_delete = rep(1,length(unique(rownames(which(simil >= cutoff_lower & simil <= cutoff_upper, arr.ind = TRUE))))))
|
|
|
|
dupe_delete = rep(1,length(unique(rownames(which(simil >= cutoff_lower & simil <= cutoff_upper, arr.ind = TRUE))))))
|
|
|
|
bulk <- c(apply(df, 1, bulk_writer, varname='duplicates', type = 'set', ver = ver),
|
|
|
|
bulk <- c(apply(df, 1, bulk_writer, varname='duplicates', type = 'set', ver = ver),
|
|
|
|
apply(dupe_delete, 1, bulk_writer, varname='_delete', type = 'set', ver = ver))
|
|
|
|
apply(dupe_delete, 1, bulk_writer, varname='_delete', type = 'set', ver = ver))
|
|
|
|
if (length(bulk) > 0) {
|
|
|
|
|
|
|
|
res <- elastic_update(bulk, es_super = es_super, localhost = localhost)
|
|
|
|
res <- elastic_update(bulk, es_super = es_super, localhost = localhost)
|
|
|
|
}
|
|
|
|
|
|
|
|
return(paste0('Checked ',params$doctypes,' on ',params$dates ))
|
|
|
|
return(paste0('Checked ',params$doctypes,' on ',params$dates ))
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
return(paste0('No duplicates for ',params$doctypes,' on ',params$dates ))
|
|
|
|
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
return(paste0('No results for ',params$doctypes,' on ',params$dates ))
|
|
|
|
return(paste0('No results for ',params$doctypes,' on ',params$dates ))
|
|
|
|
}
|
|
|
|
}
|
|
|
|