actorizer: Removed nested parallelization function

query_gen_actors: Integrated startDate and endDate for parties, changed party exception method from abbreviation only to both full names and abbreviations for NL and BE
master
Your Name 4 years ago
parent 5de4e1488c
commit b406304c80

@ -14,13 +14,28 @@
#' @examples #' @examples
#' actorizer(out, localhost = F, ids, prefix, postfix, identifier, es_super) #' actorizer(out, localhost = F, ids, prefix, postfix, identifier, es_super)
actorizer <- function(out, localhost = F, ids, prefix, postfix, pre_tags, post_tags, es_super, ver) { actorizer <- function(out, localhost = F, ids, prefix, postfix, pre_tags, post_tags, es_super, ver) {
sentencizer <- function(row, out, ids, prefix, postfix, pre_tags, post_tags, pre_tags_regex, post_tags_regex) { offsetter <- function(x, pre_tags, post_tags) {
doc <- out[row,] return(as.list(as.data.frame(x-((row(x)-1)*(nchar(pre_tags)+nchar(post_tags))))))
if (sum(nchar(doc$merged) > 990000)) { }
out <- mamlr:::out_parser(out, field = 'highlight', clean = F)
prefix[prefix==''] <- NA
postfix[postfix==''] <- NA
pre_tags_regex <- gsub("([.|()\\^{}+$*?]|\\[|\\])", "\\\\\\1", pre_tags)
post_tags_regex <- gsub("([.|()\\^{}+$*?]|\\[|\\])", "\\\\\\1", post_tags)
out$markers <- lapply(str_locate_all(out$merged,coll(pre_tags)), offsetter, pre_tags = pre_tags, post_tags = post_tags)
markers <- out %>%
select(`_id`,markers) %>%
unnest_wider(markers) %>%
rename(marker_start = start, marker_end = end) %>%
unnest(colnames(.))
if (sum(nchar(out$merged) > 990000) > 0) {
stop("One or more documents in this batch exceed 990000 characters") stop("One or more documents in this batch exceed 990000 characters")
} }
# Extracting ud output from document # Extracting ud output from document
ud <- doc %>% ud <- out %>%
select(`_id`,`_source.ud`, merged) %>% select(`_id`,`_source.ud`, merged) %>%
unnest(cols = c("_source.ud")) %>% unnest(cols = c("_source.ud")) %>%
select(`_id`,lemma,start,end, sentence_id,merged) %>% select(`_id`,lemma,start,end, sentence_id,merged) %>%
@ -96,44 +111,17 @@ actorizer <- function(out, localhost = F, ids, prefix, postfix, pre_tags, post_t
sentence_count = first(sentence_count)# List of actor ids sentence_count = first(sentence_count)# List of actor ids
) %>% ) %>%
mutate( mutate(
prom = occ/sentence_count, # Relative prominence of actor in article (number of occurences/total # sentences) prom = occ/sentence_count, # Relative prominence of actor in article (number of occurrences/total # sentences)
rel_first = 1-(first/sentence_count), # Relative position of first occurence at sentence level rel_first = 1-(first/sentence_count), # Relative position of first occurrence at sentence level
) %>% ) %>%
select(`_id`:occ, prom,rel_first,first,ids) select(`_id`:occ, prom,rel_first,first,ids)
return(hits)
}
out <- mamlr:::out_parser(out, field = 'highlight', clean = F)
offsetter <- function(x, pre_tags, post_tags) {
return(as.list(as.data.frame(x-((row(x)-1)*(nchar(pre_tags)+nchar(post_tags))))))
}
prefix[prefix==''] <- NA
postfix[postfix==''] <- NA
pre_tags_regex <- gsub("([.|()\\^{}+$*?]|\\[|\\])", "\\\\\\1", pre_tags)
post_tags_regex <- gsub("([.|()\\^{}+$*?]|\\[|\\])", "\\\\\\1", post_tags)
out$markers <- future_lapply(str_locate_all(out$merged,coll(pre_tags)), offsetter, pre_tags = pre_tags, post_tags = post_tags)
markers <- out %>%
select(`_id`,markers) %>%
unnest_wider(markers) %>%
rename(marker_start = start, marker_end = end) %>%
unnest(colnames(.))
# ids <- fromJSON(ids) if (nrow(hits) == 0) {
updates <- sentencizer(1:1024,
out = out,
ids = ids,
postfix = postfix,
prefix=prefix,
pre_tags_regex = pre_tags_regex,
pre_tags = pre_tags,
post_tags_regex = post_tags_regex,
post_tags = post_tags)
if (nrow(updates) == 0) {
print("Nothing to update for this batch") print("Nothing to update for this batch")
return(NULL) return(NULL)
} else { } else {
bulk <- apply(updates, 1, bulk_writer, varname ='actorsDetail', type = 'add', ver = ver) bulk <- apply(hits, 1, bulk_writer, varname ='actorsDetail', type = 'add', ver = ver)
bulk <- c(bulk,apply(updates[c(1,11)], 1, bulk_writer, varname='actors', type = 'add', ver = ver)) bulk <- c(bulk,apply(hits[c(1,11)], 1, bulk_writer, varname='actors', type = 'add', ver = ver))
return(elastic_update(bulk, es_super = es_super, localhost = localhost)) return(elastic_update(bulk, es_super = es_super, localhost = localhost))
} }

@ -136,8 +136,8 @@ query_gen_actors <- function(actor, country, pre_tags, post_tags) {
### Query generation for party searches ### Query generation for party searches
if (actor$`_source.function` == "Party") { if (actor$`_source.function` == "Party") {
actor$`_source.startDate` <- "2000-01-01" # actor$`_source.startDate` <- "2000-01-01"
actor$`_source.endDate` <- "2099-01-01" # actor$`_source.endDate` <- "2099-01-01"
if (nchar(actor$`_source.partyNameSearchShort`[[1]]) > 0) { if (nchar(actor$`_source.partyNameSearchShort`[[1]]) > 0) {
# If uk, no or dk, search for both regular abbreviations, and genitive forms # If uk, no or dk, search for both regular abbreviations, and genitive forms
if (country == "uk" | country == "no" | country == "dk") { if (country == "uk" | country == "no" | country == "dk") {
@ -167,7 +167,7 @@ query_gen_actors <- function(actor, country, pre_tags, post_tags) {
ids <- str_c(actor$`_source.partyId`,'_f') ids <- str_c(actor$`_source.partyId`,'_f')
actorid <- str_c(actor$`_source.partyId`,'_f') actorid <- str_c(actor$`_source.partyId`,'_f')
query <- generator(country, actor$`_source.startDate`, actor$`_source.endDate`, query_string, pre_tags, post_tags, actorid) query <- generator(country, actor$`_source.startDate`, actor$`_source.endDate`, query_string, pre_tags, post_tags, actorid)
if (country == 'uk') { if (country == 'uk' | country == 'nl' | country == 'be') {
df2 <- data.frame(query = query, ids = I(ids), prefix = actor$`_source.notPrecededBy`, postfix = actor$`_source.notFollowedBy`, stringsAsFactors = F) df2 <- data.frame(query = query, ids = I(ids), prefix = actor$`_source.notPrecededBy`, postfix = actor$`_source.notFollowedBy`, stringsAsFactors = F)
} else { } else {
df2 <- data.frame(query = query, ids = I(ids), prefix = NA, postfix = NA, stringsAsFactors = F) df2 <- data.frame(query = query, ids = I(ids), prefix = NA, postfix = NA, stringsAsFactors = F)

Loading…
Cancel
Save